/* global React */ { const { useState, useEffect, useRef, useMemo, useCallback, createContext, useContext } = React; // ========================================================================= // VOICE AGENT CONTEXT // ========================================================================= // State machine: idle | listening | speaking | thinking | saving | confirm | disconnected | reviewing const VoiceCtx = createContext(null); function useVoice() { return useContext(VoiceCtx); } const LOCAL_LABELS = { company_legal_name: 'Company legal name', company_type: 'Company type', representative_type: 'Owner/operator status', underlying_facility_owner: 'Underlying facility owner', contact_full_name: 'Full name', contact_email: 'Work email', contact_phone_or_whatsapp: 'Phone or WhatsApp', facility_name: 'Facility name', facility_address: 'Facility address', facility_city: 'Facility city', facility_country: 'Facility country', facility_stage: 'Facility stage', exclusivity_restrictions: 'Exclusivity restrictions', total_site_power_mw: 'Total site power', commissioned_power_mw: 'Commissioned power', current_it_load_mw: 'Current IT load', vacant_it_load_mw: 'Vacant IT load', power_available_0_90_days_mw: 'Power available in 0-90 days', power_available_3_6_months_mw: 'Power available in 3-6 months', power_available_6_12_months_mw: 'Power available in 6-12 months', firm_expansion_path_30mw_18mo: '30 MW expansion path', utility_provider: 'Utility provider', substation_feeder_detail: 'Substation or feeder detail', power_redundancy: 'Power redundancy', power_cost_per_kwh: 'Power cost per kWh', max_capacity_discussable_mw: 'Maximum discussable capacity', available_whitespace: 'Available whitespace', floor_loading_kg_per_m2: 'Floor loading', cooling_type: 'Cooling type', current_rack_density_kw: 'Current rack density', max_rack_density_after_upgrade_kw: 'Max rack density after upgrades', liquid_cooling_readiness: 'Liquid cooling readiness', cooling_upgrade_timeline: 'Cooling upgrade timeline', water_availability_or_constraints: 'Water availability or constraints', heat_rejection_system: 'Heat rejection system', cooling_expansion_constraints: 'Cooling or expansion constraints', network_carriers: 'Available carriers', redundant_fiber_paths: 'Redundant fiber paths', current_bandwidth_capacity: 'Current bandwidth capacity', infiniband_roce_ndr_support: 'InfiniBand / RoCE / NDR support', physical_security_model: 'Physical security model', onsite_staff_24_7: '24/7 onsite staff', remote_hands_available: 'Remote hands', willing_host_neevcloud_gpus: 'Host NeevCloud-owned GPUs', willing_finance_gpus: 'Finance GPUs', preferred_commercial_model: 'Preferred commercial model', open_to_first_right_refusal: 'First right of refusal', permits_current_and_phase2: 'Permits', major_upgrades_required: 'Major upgrades required', agreement_to_ready_timeline: 'Agreement to ready capacity timeline', }; const QUESTION_PROMPTS = { 'company.legal_name': 'Let\'s start simple. What is the company\'s legal name?', 'company.company_type': 'How would you describe the company: operator, colocation provider, real estate owner, telco, cloud/neocloud, broker, or something else?', 'company.representative_type': 'Are you the direct owner or operator, or an authorized representative?', 'company.underlying_facility_owner': 'Who is the underlying facility owner? If this does not apply, say not applicable.', 'contact.full_name': 'Who should NeevCloud contact about this IDN DC partner registration?', 'contact.email': 'What work email should NeevCloud use for follow-up?', 'contact.phone_or_whatsapp': 'What phone or WhatsApp number should NeevCloud use?', 'facility.name': 'What facility name or internal site code should we use?', 'facility.address': 'What is the full facility address?', 'facility.city': 'Which city is the facility in?', 'facility.country': 'Which country is the facility in?', 'facility.stage': 'What stage is the facility at: land, shell, powered shell, fitted data hall, or live facility?', 'facility.exclusivity_restrictions': 'Are there any hyperscaler, anchor tenant, or exclusivity restrictions?', 'power.total_site_power_mw': 'How many megawatts is the total site power capacity?', 'power.commissioned_power_mw': 'How many megawatts of power are currently commissioned?', 'power.current_it_load_mw': 'What is the current IT load in megawatts?', 'power.vacant_it_load_mw': 'How much IT load is currently vacant or available, in megawatts?', 'power.available_0_90_days_mw': 'How much power can be available in the next 0 to 90 days?', 'power.available_3_6_months_mw': 'How much power can be available in 3 to 6 months?', 'power.available_6_12_months_mw': 'How much power can be available in 6 to 12 months?', 'power.firm_expansion_path_30mw_18mo': 'Is there a firm path to the next 30 MW within 18 months?', 'power.utility_provider': 'Who is the utility or power provider?', 'power.substation_feeder_detail': 'What is the substation or feeder detail?', 'power.power_redundancy': 'What is the power redundancy setup?', 'power.estimated_power_cost_per_kwh': 'What is the estimated power cost per kilowatt-hour?', 'power.max_capacity_discussable_mw': 'What is the maximum capacity you are willing to discuss with NeevCloud?', 'space.available_whitespace': 'How much available whitespace is there in square feet?', 'structural.floor_loading_kg_per_m2': 'What is the floor loading limit in kilograms per square meter?', 'cooling.cooling_type': 'What cooling setup does the facility currently use?', 'cooling.current_rack_density_kw': 'What rack density is supported today, in kilowatts per rack?', 'cooling.max_rack_density_after_upgrade_kw': 'What rack density could be supported after upgrades?', 'cooling.liquid_cooling_readiness': 'What is the facility\'s liquid cooling readiness?', 'cooling.cooling_upgrade_timeline': 'What is the cooling upgrade timeline?', 'cooling.water_availability_or_constraints': 'What water availability or constraints should NeevCloud know about?', 'cooling.heat_rejection_system': 'What heat rejection system is in place?', 'cooling.expansion_constraints': 'Are there cooling or physical expansion constraints?', 'network.carriers': 'Which network carriers are available at the site?', 'network.redundant_fiber_paths': 'Are redundant fiber paths available?', 'network.current_bandwidth_capacity': 'What is the current bandwidth capacity?', 'network.infiniband_roce_ndr_support': 'Can the site support InfiniBand, RoCE, or NDR networking?', 'ops.physical_security_model': 'What is the physical security model?', 'ops.onsite_staff_24_7': 'Is there 24/7 onsite staff?', 'ops.remote_hands_available': 'Are remote hands available?', 'commercial.willing_host_neevcloud_gpus': 'Are you willing to host NeevCloud-owned GPUs?', 'commercial.willing_finance_gpus': 'Are you willing to finance GPUs yourself?', 'commercial.preferred_commercial_model': 'What commercial model do you prefer?', 'commercial.open_to_first_right_refusal': 'Are you open to first right of refusal or reserved capacity for NeevCloud?', 'deployment.permits_current_and_phase2': 'What permits are required for GPU deployment or Phase 2 capacity?', 'deployment.major_upgrades_required': 'What major upgrades are needed before AI deployment?', 'deployment.agreement_to_ready_timeline': 'What is your estimated timeline from agreement to deployment-ready capacity?', }; const VOICE_DEFAULT_DOCUMENT_FILE_CATEGORY = 'site_documents'; const SCRATCH_FIRST_MESSAGE = 'Hi, I am NeevCloud\'s AI intake agent. I can help complete this IDN DC partner registration and save answers into the visible form for your review.'; const RESUME_FIRST_MESSAGE = 'Hi, ready to resume the intake?'; const REVIEW_FIRST_MESSAGE = 'Ready to review your answers?'; function getQuestionField(questionIndex, questionKey) { return questionIndex?.fieldsByQuestion?.[questionKey] || null; } function schemaQuestionKeys(questionIndex, requiredOnly = false) { const keys = requiredOnly ? questionIndex?.requiredQuestionKeys : questionIndex?.allQuestionKeys; if (keys?.length) return keys; return Object.keys(QUESTION_PROMPTS); } function fieldOptions(field) { return (field?.options || []) .map((option) => option.label || option.value) .filter(Boolean); } function labelForQuestion(questionKey, questionIndex = null, backendKeyMap = {}) { const field = getQuestionField(questionIndex, questionKey); if (field?.label) return field.label; const localKey = backendKeyMap[questionKey] || questionKey; return LOCAL_LABELS[localKey] || String(questionKey).split('.').slice(-1)[0].replace(/_/g, ' '); } function localKeyForQuestion(questionKey, backendKeyMap = {}, fileQuestionMap = {}) { return backendKeyMap[questionKey] || fileQuestionMap[questionKey] || questionKey; } function sectionForQuestion(questionKey, questionSectionMap = {}, questionIndex = null) { const schemaSection = questionIndex?.sectionByQuestion?.[questionKey]; if (schemaSection) return schemaSection; const prefix = String(questionKey).split('.')[0]; return questionSectionMap[prefix] || 'company'; } function promptForQuestion(questionKey, questionIndex, backendKeyMap) { const field = getQuestionField(questionIndex, questionKey); if (!field) return QUESTION_PROMPTS[questionKey] || `Let's capture ${labelForQuestion(questionKey, questionIndex, backendKeyMap)}.`; const label = field.label; const options = fieldOptions(field); if (field.field_type === 'number') return `${label}? Please answer with just the number, without units.`; if (field.field_type === 'date') return `${label}? Please give the date if known.`; if ((field.field_type === 'select' || field.field_type === 'multiselect') && options.length) { return `${label}? Options: ${options.join(', ')}.`; } if (field.field_type === 'multiselect_freeform') return `${label}? You can list multiple values.`; if (field.field_type === 'file') return `${label}? You can upload or paste a link.`; return `${label}?`; } function mapPending(item, backendKeyMap, fileQuestionMap, questionIndex = null) { return { questionKey: item.question_key, key: localKeyForQuestion(item.question_key, backendKeyMap, fileQuestionMap), label: labelForQuestion(item.question_key, questionIndex, backendKeyMap), oldValue: item.current_value, newValue: item.candidate_value, note: item.evidence || 'The voice agent heard a different value.', }; } function normalizeToolParams(raw = {}) { if (raw.parameters && typeof raw.parameters === 'object') return raw.parameters; return raw; } function valueIsEmpty(value) { return value === undefined || value === null || value === '' || (Array.isArray(value) && value.length === 0); } function answersMatch(left, right) { if (Array.isArray(left) || Array.isArray(right)) return JSON.stringify(left || []) === JSON.stringify(right || []); return left === right || (!valueIsEmpty(left) && !valueIsEmpty(right) && String(left).trim() === String(right).trim()); } function voiceProgressForAnswers(answers, backendKeyMap, fileQuestionMap, questionSectionMap, justSavedQuestionKey = null, questionIndex = null) { const required = schemaQuestionKeys(questionIndex, true); const missing = required.filter((questionKey) => { const localKey = localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap); return valueIsEmpty(answers[localKey]); }); const nextKey = missing.find((questionKey) => questionKey !== justSavedQuestionKey) || missing[0] || null; return { next_question_key: nextKey, next_question_label: nextKey ? labelForQuestion(nextKey, questionIndex, backendKeyMap) : null, next_question_text: nextKey ? promptForQuestion(nextKey, questionIndex, backendKeyMap) : null, next_section: nextKey ? sectionForQuestion(nextKey, questionSectionMap, questionIndex) : null, completion_count: required.length - missing.length, required_count: required.length, missing_required_count: missing.length, ready_for_review: missing.length === 0, }; } function visibleFormSnapshot(answers, backendKeyMap, fileQuestionMap, questionIndex = null) { const values = {}; const filled = []; const allKeys = schemaQuestionKeys(questionIndex, false); allKeys.forEach((questionKey) => { const localKey = localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap); const value = answers?.[localKey]; if (!valueIsEmpty(value)) { filled.push(questionKey); values[questionKey] = value; } }); const missing = schemaQuestionKeys(questionIndex, true).filter((questionKey) => { const localKey = localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap); return valueIsEmpty(answers?.[localKey]); }); return { filled_question_keys: filled, missing_required_question_keys: missing, answer_values: values, }; } function backendFilledQuestionKeys(state) { const keys = new Set(); (state?.filled_question_keys || []).forEach((questionKey) => { if (questionKey) keys.add(questionKey); }); Object.entries(state?.answer_values || {}).forEach(([questionKey, value]) => { if (!valueIsEmpty(value)) keys.add(questionKey); }); (state?.field_states || []).forEach((row) => { if (!row?.question_key || valueIsEmpty(row.current_value)) return; if (['answered', 'verified', 'needs_confirmation'].includes(row.status)) keys.add(row.question_key); }); return keys; } function chooseFirstMessageForVoiceSession(state, answers, backendKeyMap, fileQuestionMap, questionIndex) { const visible = visibleFormSnapshot(answers || {}, backendKeyMap, fileQuestionMap, questionIndex); const filled = backendFilledQuestionKeys(state); visible.filled_question_keys.forEach((questionKey) => filled.add(questionKey)); const mode = filled.size === 0 ? 'scratch' : (state?.ready_for_review ? 'review' : 'resume'); const message = mode === 'scratch' ? SCRATCH_FIRST_MESSAGE : (mode === 'review' ? REVIEW_FIRST_MESSAGE : RESUME_FIRST_MESSAGE); return { mode, message, filled_count: filled.size, visible_filled_count: visible.filled_question_keys.length, backend_filled_count: backendFilledQuestionKeys(state).size, }; } function buildVoiceFormContract(formSchema, questionIndex, backendKeyMap, fileQuestionMap, questionSectionMap) { const allKeys = schemaQuestionKeys(questionIndex, false); return { schema_version: formSchema?.schema_version || questionIndex?.schemaVersion || null, title: formSchema?.title || 'NeevCloud IDN DC partner intake', required_question_keys: schemaQuestionKeys(questionIndex, true), fields: allKeys.map((questionKey) => { const field = getQuestionField(questionIndex, questionKey); return { question_key: questionKey, local_key: localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap), label: field?.label || labelForQuestion(questionKey, questionIndex, backendKeyMap), field_type: field?.field_type || 'text', value_type: field?.value_type || 'string', unit: field?.unit || null, answer_format: field?.answer_format || null, examples: field?.examples || [], required: Boolean(field?.required), options: field?.options || [], section: sectionForQuestion(questionKey, questionSectionMap, questionIndex), }; }), }; } function describeVoiceError(error) { if (!error) return 'Voice session error.'; return error.closeReason || error.reason || error.message || String(error); } function describeVoiceDisconnect(details) { const message = describeVoiceError(details); if (message && message !== 'Voice session error.') return message; if (details?.reason === 'agent') return 'The AI voice agent ended the conversation.'; if (details?.reason === 'error') return 'Audio connection dropped.'; return 'Audio connection disconnected.'; } async function unlockConversationAudio(conversation) { conversation?.setVolume?.({ volume: 1 }); const output = conversation?.output; if (!output) return; if (output.context?.state !== 'running') { await output.context?.resume?.(); } const audioElement = output.audioElement; if (output.gain && output.context?.destination && !output.__neevDirectOutputConnected) { output.gain.connect(output.context.destination); output.__neevDirectOutputConnected = true; } if (audioElement) { audioElement.muted = Boolean(output.__neevDirectOutputConnected); audioElement.volume = output.__neevDirectOutputConnected ? 0 : 1; audioElement.playsInline = true; await audioElement.play?.(); } } function volumeFromFrequencyData(buffer) { if (!buffer || !buffer.length) return 0; let total = 0; for (const value of buffer) total += value / 255; return Math.min(1, total / buffer.length); } function barsFromFrequencyData(buffer, count = 18) { if (!buffer || !buffer.length) { return Array.from({ length: count }, () => 0.08); } const segment = Math.max(1, Math.floor(buffer.length / count)); return Array.from({ length: count }, (_, index) => { let total = 0; const start = index * segment; const end = Math.min(buffer.length, start + segment); for (let cursor = start; cursor < end; cursor += 1) total += buffer[cursor] / 255; return Math.min(1, (total / Math.max(1, end - start)) * 3.2); }); } function barsFromPcmBase64(audioBase64, count = 18) { if (!audioBase64 || typeof atob !== 'function') return barsFromFrequencyData(null, count); try { const binary = atob(audioBase64); const sampleCount = Math.floor(binary.length / 2); if (!sampleCount) return barsFromFrequencyData(null, count); const segment = Math.max(1, Math.floor(sampleCount / count)); return Array.from({ length: count }, (_, index) => { const start = index * segment; const end = Math.min(sampleCount, start + segment); let total = 0; let samples = 0; for (let sampleIndex = start; sampleIndex < end; sampleIndex += 1) { const lo = binary.charCodeAt(sampleIndex * 2); const hi = binary.charCodeAt(sampleIndex * 2 + 1); let sample = (hi << 8) | lo; if (sample & 0x8000) sample -= 0x10000; total += Math.abs(sample) / 32768; samples += 1; } return Math.min(1, Math.max(0.04, (total / Math.max(1, samples)) * 7)); }); } catch { return barsFromFrequencyData(null, count); } } function levelFromBars(bars = []) { if (!bars.length) return 0; return bars.reduce((sum, level) => sum + level, 0) / bars.length; } function VoiceProvider({ children, setAnswer, setBackendAnswer, answers, scrollToSection, getVoiceState, getVoiceToken, registerVoiceConversation, proposeVoiceAnswer, confirmVoiceOverwrite, markVoiceUnresolved, prepareVoiceReview, resetVoiceIntake, uploadArtifact, attachArtifactLink, backendKeyMap = {}, fileQuestionMap = {}, questionSectionMap = {}, normalizeBackendAnswer, validateBackendAnswer, formSchema = null, questionIndex = null, }) { const [phase, setPhase] = useState('idle'); // idle, listening, speaking, thinking, saving, confirm, disconnected, reviewing, education const [currentIdx, setCurrentIdx] = useState(0); const [transcript, setTranscript] = useState([]); const [activity, setActivity] = useState([{ key: 'state', label: 'Voice agent ready', value: 'Form updates instantly; backend syncs in the background.', ts: Date.now(), kind: 'info' }]); const [pendingConfirm, setPendingConfirm] = useState(null); const [followUps, setFollowUps] = useState([]); const [needsConfirm, setNeedsConfirm] = useState([]); const [highlight, setHighlight] = useState(null); const [education, setEducation] = useState(null); const [uploads, setUploads] = useState([]); const [running, setRunning] = useState(false); const [voiceState, setVoiceState] = useState(null); const [connectionError, setConnectionError] = useState(''); const [audioBars, setAudioBars] = useState(() => barsFromFrequencyData(null)); const fieldRefs = useRef(new Map()); const answersRef = useRef(answers || {}); const pendingConfirmRef = useRef(null); const conversationRef = useRef(null); const manualDisconnectRef = useRef(false); const audioChunkCountRef = useRef(0); const lastAgentAudioAtRef = useRef(0); const lastAgentBarsRef = useRef(barsFromFrequencyData(null)); const latencyRef = useRef({ turnId: 0, userTranscriptAt: null, firstToolSeenForTurn: false, toolResultAt: null, waitingForAgentText: false, waitingForAgentAudio: false, }); const setPendingConfirmState = useCallback((nextPending) => { pendingConfirmRef.current = nextPending; setPendingConfirm(nextPending); }, []); useEffect(() => { answersRef.current = answers || {}; }, [answers]); const logLatency = useCallback((event, data = {}) => { const now = Math.round(performance?.now?.() || Date.now()); const trace = latencyRef.current; const entry = { event, t_ms: now, turn_id: trace.turnId, since_user_transcript_ms: trace.userTranscriptAt ? now - trace.userTranscriptAt : null, since_tool_result_ms: trace.toolResultAt ? now - trace.toolResultAt : null, ...data, }; window.__NEEV_VOICE_LATENCY_LOGS = [...(window.__NEEV_VOICE_LATENCY_LOGS || []), entry].slice(-200); console.info('[voice-latency]', entry); return entry; }, []); const logToolEvent = useCallback((event, data = {}) => { const entry = { event, ts: Date.now(), ...data }; window.__NEEV_VOICE_TOOL_LOGS = [...(window.__NEEV_VOICE_TOOL_LOGS || []), entry].slice(-300); console.info('[voice-tool]', entry); return entry; }, []); const noteToolCall = useCallback((toolName, params = null) => { logToolEvent('call', { tool: toolName, params }); if (!latencyRef.current.firstToolSeenForTurn) { latencyRef.current.firstToolSeenForTurn = true; logLatency('first_client_tool_call_received', { tool: toolName }); } logLatency('client_tool_call_received', { tool: toolName }); }, [logLatency, logToolEvent]); const noteToolResult = useCallback((toolName, data = {}) => { latencyRef.current.toolResultAt = Math.round(performance?.now?.() || Date.now()); latencyRef.current.waitingForAgentText = true; latencyRef.current.waitingForAgentAudio = true; logToolEvent('result', { tool: toolName, result: data }); logLatency('client_tool_result_returned', { tool: toolName, ...data }); }, [logLatency, logToolEvent]); const registerField = useCallback((key, el) => { if (el) fieldRefs.current.set(key, el); else fieldRefs.current.delete(key); }, []); const flashField = useCallback((keyOrQuestionKey) => { const localKey = localKeyForQuestion(keyOrQuestionKey, backendKeyMap, fileQuestionMap); setHighlight(localKey); const el = fieldRefs.current.get(localKey); if (el && scrollToSection) { const sec = el.closest('[id^="sec-"]'); if (sec) sec.scrollIntoView({ behavior: 'smooth', block: 'start' }); } setTimeout(() => setHighlight((h) => (h === localKey ? null : h)), 2400); }, [backendKeyMap, fileQuestionMap, scrollToSection]); const syncVoiceState = useCallback((state) => { if (!state) return state; const hydrated = []; const hydrateAnswer = (questionKey, value, source) => { if (!questionKey || valueIsEmpty(value)) return; const localKey = localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap); if (!localKey || localKey === questionKey) return; const currentValue = answersRef.current?.[localKey]; if (answersMatch(currentValue, value)) return; setBackendAnswer?.(questionKey, value); hydrated.push({ question_key: questionKey, source }); }; Object.entries(state.answer_values || {}).forEach(([questionKey, value]) => { hydrateAnswer(questionKey, value, 'answer_values'); }); (state.field_states || []).forEach((row) => { if (!['answered', 'verified'].includes(row.status)) return; hydrateAnswer(row.question_key, row.current_value, 'field_states'); }); if (hydrated.length) logToolEvent('visible_hydrated_from_backend_state', { question_keys: hydrated }); setVoiceState(state); const pending = (state.pending_confirmations || []).map((item) => mapPending(item, backendKeyMap, fileQuestionMap, questionIndex)); if (pending.length || !pendingConfirmRef.current) { setNeedsConfirm(pending); setPendingConfirmState(pending[0] || null); } setFollowUps((state.unresolved_question_keys || []).map((questionKey) => ({ questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), note: 'Marked unresolved for NeevCloud follow-up.', }))); return state; }, [backendKeyMap, fileQuestionMap, logToolEvent, questionIndex, setBackendAnswer, setPendingConfirmState]); const refreshState = useCallback(async () => { if (!getVoiceState) return null; const state = await getVoiceState(); return syncVoiceState(state); }, [getVoiceState, syncVoiceState]); useEffect(() => { if (!running || !getVoiceState) return undefined; const timer = setInterval(() => { refreshState().catch((error) => { logToolEvent('state_refresh_error', { message: error?.message || String(error) }); }); }, 3000); return () => clearInterval(timer); }, [running, getVoiceState, refreshState, logToolEvent]); useEffect(() => { if (!running) { const idleBars = barsFromFrequencyData(null); lastAgentAudioAtRef.current = 0; lastAgentBarsRef.current = idleBars; setAudioBars(idleBars); return undefined; } const timer = setInterval(() => { const conversation = conversationRef.current; if (!conversation) return; const outputData = conversation.getOutputByteFrequencyData?.(); const inputData = conversation.getInputByteFrequencyData?.(); const nextOutputLevel = volumeFromFrequencyData(outputData); const nextInputLevel = volumeFromFrequencyData(inputData); const outputBars = barsFromFrequencyData(outputData); const inputBars = barsFromFrequencyData(inputData); const recentAgentAudio = Date.now() - lastAgentAudioAtRef.current < 700; const decodedAgentBars = lastAgentBarsRef.current; if (recentAgentAudio && levelFromBars(decodedAgentBars) > 0.045) { setAudioBars(decodedAgentBars); } else if (nextOutputLevel > 0.015) { setAudioBars(outputBars); } else if (nextInputLevel > 0.015) { setAudioBars(inputBars); } else { setAudioBars(barsFromFrequencyData(null)); } }, 80); return () => clearInterval(timer); }, [running]); const questionFlow = useMemo(() => { const keys = voiceState?.missing_required_question_keys?.length ? voiceState.missing_required_question_keys : schemaQuestionKeys(questionIndex, true); return keys.map((questionKey) => ({ questionKey, key: localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap), q: promptForQuestion(questionKey, questionIndex, backendKeyMap), section: sectionForQuestion(questionKey, questionSectionMap, questionIndex), })); }, [voiceState, backendKeyMap, fileQuestionMap, questionSectionMap, questionIndex]); const addActivity = useCallback((item) => { setActivity((a) => [{ ts: Date.now(), ...item }, ...a].slice(0, 10)); }, []); const pushTranscript = useCallback((who, text) => { setTranscript((t) => [...t, { who, text, ts: Date.now() }].slice(-8)); }, []); const applyToolState = useCallback((result, questionKey, actionLabel) => { if (result?.voice_state) syncVoiceState(result.voice_state); if (questionKey) { flashField(questionKey); addActivity({ key: questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), value: actionLabel || result?.action || 'saved', ts: Date.now(), kind: result?.action || 'saved', }); } return result; }, [addActivity, backendKeyMap, flashField, questionIndex, syncVoiceState]); const saveAnswerAndAdvance = useCallback((raw) => { const params = normalizeToolParams(raw); const questionKey = params.question_key; const rawAnswerValue = params.answer_value; noteToolCall('saveAnswerAndAdvance', params); if (!questionKey || valueIsEmpty(rawAnswerValue)) { const result = { ok: false, action: 'error', message: 'question_key and answer_value are required.', question_key: questionKey || null, }; noteToolResult('saveAnswerAndAdvance', { action: result.action }); return result; } const localKey = localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap); if (!localKey || localKey === questionKey) { const result = { ok: false, action: 'error', message: `No visible form field is mapped for ${questionKey}.`, question_key: questionKey, }; noteToolResult('saveAnswerAndAdvance', { action: result.action, question_key: questionKey }); return result; } const answerValue = normalizeBackendAnswer ? normalizeBackendAnswer(questionKey, rawAnswerValue) : rawAnswerValue; const validation = validateBackendAnswer ? validateBackendAnswer(questionKey, answerValue) : { ok: true }; if (!validation.ok) { const result = { ok: false, optimistic: false, action: 'invalid_format', message: validation.message || 'That answer does not match the expected field format.', question_key: questionKey, expected_format: getQuestionField(questionIndex, questionKey)?.answer_format || null, }; addActivity({ key: questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), value: 'format needs clarification', kind: 'error', }); setPhase('listening'); noteToolResult('saveAnswerAndAdvance', { action: result.action, question_key: questionKey }); return result; } const currentAnswers = answersRef.current || {}; const currentValue = currentAnswers[localKey]; const progressBefore = voiceProgressForAnswers(currentAnswers, backendKeyMap, fileQuestionMap, questionSectionMap, questionKey, questionIndex); if (!valueIsEmpty(currentValue) && !answersMatch(currentValue, answerValue)) { const conflict = { questionKey, key: localKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), oldValue: currentValue, newValue: answerValue, note: params.evidence || 'The voice agent heard a different value.', }; setPendingConfirmState(conflict); setNeedsConfirm((items) => [conflict, ...items.filter((item) => item.questionKey !== questionKey)]); setPhase('confirm'); flashField(questionKey); addActivity({ key: questionKey, label: conflict.label, value: 'needs confirmation', kind: 'needs_confirmation', }); logLatency('backend_save_request_start', { question_key: questionKey, action: 'persist_conflict' }); Promise.resolve(proposeVoiceAnswer?.({ question_key: questionKey, answer_value: answerValue, confidence: params.confidence ?? 0.75, evidence: params.evidence || 'Captured by the voice survey.', source: params.source || 'voice_agent', conversation_id: params.conversation_id, })).then((backendResult) => { logLatency('backend_save_request_end', { question_key: questionKey, action: backendResult?.action || 'unknown' }); logLatency('backend_response_received', { question_key: questionKey, action: backendResult?.action || 'unknown' }); const pendingStillCurrent = pendingConfirmRef.current?.questionKey === questionKey && answersRef.current?.[localKey] === currentValue; if (backendResult?.voice_state && pendingStillCurrent) syncVoiceState(backendResult.voice_state); }).catch((error) => { logLatency('backend_save_request_end', { question_key: questionKey, action: 'error', error: error?.message || String(error) }); addActivity({ key: questionKey, label: conflict.label, value: `confirmation sync pending: ${error?.message || 'network error'}`, kind: 'error', }); }); const result = { ok: true, optimistic: false, action: 'needs_confirmation', message: 'Existing answer differs. Ask for confirmation before overwriting.', question_key: questionKey, current_value: currentValue, candidate_value: answerValue, ...progressBefore, }; noteToolResult('saveAnswerAndAdvance', { action: result.action, question_key: questionKey }); return result; } const nextAnswers = { ...currentAnswers, [localKey]: answerValue }; answersRef.current = nextAnswers; const localApplied = setBackendAnswer ? setBackendAnswer(questionKey, answerValue, { markSynced: false }) : (setAnswer?.(localKey, answerValue), true); const progress = voiceProgressForAnswers(nextAnswers, backendKeyMap, fileQuestionMap, questionSectionMap, questionKey, questionIndex); const section = sectionForQuestion(questionKey, questionSectionMap, questionIndex); logLatency('frontend_optimistic_form_update', { question_key: questionKey, local_key: localKey }); logToolEvent('visible_form_update', { question_key: questionKey, local_key: localKey, raw_value: rawAnswerValue, visible_value: answerValue }); flashField(questionKey); if (progress.next_section && scrollToSection) { scrollToSection(progress.next_section); logLatency('next_question_displayed', { question_key: progress.next_question_key, section: progress.next_section }); } else if (section && scrollToSection) { scrollToSection(section); } addActivity({ key: questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), value: 'saved instantly', kind: 'saved', }); setPhase('listening'); logLatency('backend_save_request_start', { question_key: questionKey }); Promise.resolve(proposeVoiceAnswer?.({ question_key: questionKey, answer_value: answerValue, confidence: params.confidence ?? 0.75, evidence: params.evidence || 'Captured by the voice survey.', source: params.source || 'voice_agent', conversation_id: params.conversation_id, })).then((result) => { logLatency('backend_save_request_end', { question_key: questionKey, action: result?.action || 'unknown' }); logLatency('backend_response_received', { question_key: questionKey, action: result?.action || 'unknown' }); if (result?.voice_state) syncVoiceState(result.voice_state); if (result?.action === 'needs_confirmation') { if (setBackendAnswer && result.question_key) setBackendAnswer(result.question_key, result.current_value); setPendingConfirmState(mapPending({ question_key: result.question_key, current_value: result.current_value, candidate_value: result.candidate_value, evidence: params.evidence, }, backendKeyMap, fileQuestionMap, questionIndex)); setPhase('confirm'); } }).catch((error) => { logLatency('backend_save_request_end', { question_key: questionKey, action: 'error', error: error?.message || String(error) }); addActivity({ key: questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), value: `backend save pending: ${error?.message || 'network error'}`, kind: 'error', }); }); const result = { ok: true, optimistic: true, backend_persisted: false, local_applied: localApplied, action: 'saved', message: 'Saved in the visible form. Backend persistence is running.', question_key: questionKey, current_value: answerValue, candidate_value: null, ...progress, }; noteToolResult('saveAnswerAndAdvance', { action: result.action, question_key: questionKey }); return result; }, [ addActivity, backendKeyMap, fileQuestionMap, flashField, logLatency, noteToolCall, noteToolResult, proposeVoiceAnswer, questionIndex, questionSectionMap, normalizeBackendAnswer, validateBackendAnswer, scrollToSection, setAnswer, setBackendAnswer, syncVoiceState, logToolEvent, setPendingConfirmState, ]); const confirmPendingOverwrite = useCallback(async (raw = {}, mode = 'replace') => { const params = normalizeToolParams(raw); const pending = pendingConfirmRef.current; const questionKey = params.question_key || pending?.questionKey; if (!questionKey) { return { ok: false, action: 'error', message: 'No pending overwrite to confirm.', question_key: null }; } const localKey = localKeyForQuestion(questionKey, backendKeyMap, fileQuestionMap); const rawAcceptedValue = params.accepted_value ?? (mode === 'keep' ? pending?.oldValue : pending?.newValue); const acceptedValue = normalizeBackendAnswer ? normalizeBackendAnswer(questionKey, rawAcceptedValue) : rawAcceptedValue; const validation = validateBackendAnswer ? validateBackendAnswer(questionKey, acceptedValue) : { ok: true }; if (!validation.ok) { return { ok: false, action: 'invalid_format', message: validation.message, question_key: questionKey }; } if (valueIsEmpty(acceptedValue)) { return { ok: false, action: 'error', message: 'No confirmed value was available.', question_key: questionKey }; } const nextAnswers = { ...(answersRef.current || {}), [localKey]: acceptedValue }; answersRef.current = nextAnswers; const action = mode === 'keep' ? 'kept_original' : 'overwritten'; const label = mode === 'keep' ? 'kept existing' : 'overwritten'; if (setBackendAnswer) setBackendAnswer(questionKey, acceptedValue, { markSynced: false }); else setAnswer?.(localKey, acceptedValue); setPendingConfirmState(null); setNeedsConfirm((items) => items.filter((item) => item.questionKey !== questionKey)); setPhase('listening'); flashField(questionKey); addActivity({ key: questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), value: label, kind: action, }); logLatency('frontend_optimistic_form_update', { question_key: questionKey, local_key: localKey, action }); logLatency('backend_save_request_start', { question_key: questionKey, action }); try { const result = await confirmVoiceOverwrite({ question_key: questionKey, accepted_value: acceptedValue, conversation_id: params.conversation_id, }); logLatency('backend_save_request_end', { question_key: questionKey, action: result?.action || action }); logLatency('backend_response_received', { question_key: questionKey, action: result?.action || action }); if (result?.voice_state) syncVoiceState(result.voice_state); noteToolResult('confirmOverwrite', { action: result?.action || action, question_key: questionKey }); return { ...result, optimistic: true }; } catch (error) { logLatency('backend_save_request_end', { question_key: questionKey, action: 'error', error: error?.message || String(error) }); addActivity({ key: questionKey, label: labelForQuestion(questionKey, questionIndex, backendKeyMap), value: `backend confirmation pending: ${error?.message || 'network error'}`, kind: 'error', }); const progress = voiceProgressForAnswers(nextAnswers, backendKeyMap, fileQuestionMap, questionSectionMap, questionKey, questionIndex); const result = { ok: true, optimistic: true, backend_persisted: false, action, message: 'Confirmed in the visible form. Backend confirmation is pending.', question_key: questionKey, current_value: acceptedValue, candidate_value: null, ...progress, }; noteToolResult('confirmOverwrite', { action: result.action, question_key: questionKey }); return result; } }, [ addActivity, backendKeyMap, confirmVoiceOverwrite, fileQuestionMap, flashField, logLatency, noteToolResult, questionIndex, questionSectionMap, normalizeBackendAnswer, validateBackendAnswer, setAnswer, setBackendAnswer, syncVoiceState, setPendingConfirmState, ]); const aiSave = useCallback(async (questionKey, value, label, opts = {}) => { if (!proposeVoiceAnswer) return; const normalizedValue = normalizeBackendAnswer ? normalizeBackendAnswer(questionKey, value) : value; const validation = validateBackendAnswer ? validateBackendAnswer(questionKey, normalizedValue) : { ok: true }; if (!validation.ok) { addActivity({ key: questionKey, label, value: 'format needs clarification', kind: 'error' }); pushTranscript('agent', validation.message || 'That answer does not match the expected field format.'); return; } setPhase('saving'); const result = await proposeVoiceAnswer({ question_key: questionKey, answer_value: normalizedValue, confidence: opts.confidence ?? 0.75, evidence: opts.evidence || 'Typed into the voice intake panel.', source: opts.source || 'text_input', }); applyToolState(result, questionKey); setPhase(result.action === 'needs_confirmation' ? 'confirm' : 'listening'); }, [addActivity, applyToolState, normalizeBackendAnswer, proposeVoiceAnswer, pushTranscript, validateBackendAnswer]); const acceptOverwrite = async () => { if (!pendingConfirm) return; await confirmPendingOverwrite({ question_key: pendingConfirm.questionKey }, 'replace'); }; const keepOriginal = async () => { if (!pendingConfirm) return; await confirmPendingOverwrite({ question_key: pendingConfirm.questionKey }, 'keep'); }; const markFollowUp = async () => { if (pendingConfirm) { await markVoiceUnresolved?.({ question_key: pendingConfirm.questionKey, reason: 'Conflicting answer needs operator review.', }).then((result) => applyToolState(result, pendingConfirm.questionKey, 'unresolved')); setPendingConfirmState(null); } setPhase('listening'); }; const startNewIntake = useCallback(async (raw = {}) => { const params = normalizeToolParams(raw); noteToolCall('startNewIntake', params); setPhase('saving'); try { const result = await resetVoiceIntake?.({ reason: params.reason || 'User asked the voice agent to clear the form and restart.', }); answersRef.current = {}; setPendingConfirmState(null); setNeedsConfirm([]); setFollowUps([]); if (result?.voice_state) syncVoiceState(result.voice_state); if (scrollToSection) scrollToSection('company'); addActivity({ key: 'voice-reset', label: 'Fresh draft', value: 'Visible form cleared', kind: 'saved', }); setPhase('listening'); noteToolResult('startNewIntake', { action: result?.action || 'started_new_draft' }); return result || { ok: true, action: 'started_new_draft', message: 'Visible form cleared. Fresh draft started.' }; } catch (error) { setPhase('listening'); const result = { ok: false, action: 'error', message: `Could not start a fresh draft: ${error?.message || 'unknown error'}`, }; addActivity({ key: 'voice-reset', label: 'Fresh draft', value: result.message, kind: 'error' }); noteToolResult('startNewIntake', { action: 'error' }); return result; } }, [addActivity, noteToolCall, noteToolResult, resetVoiceIntake, scrollToSection, syncVoiceState, setPendingConfirmState]); const focusQuestion = useCallback(async (raw) => { const { question_key } = normalizeToolParams(raw); if (!question_key) return { ok: false, error: 'question_key is required' }; const section = sectionForQuestion(question_key, questionSectionMap, questionIndex); if (scrollToSection && section) scrollToSection(section); flashField(question_key); return { ok: true, focused_section: section }; }, [flashField, questionIndex, questionSectionMap, scrollToSection]); const clientTools = useMemo(() => ({ saveAnswerAndAdvance, startNewIntake, getIntakeState: async () => { noteToolCall('getIntakeState'); const result = await refreshState(); const withVisibleState = result ? { ...result, visible_form: visibleFormSnapshot(answersRef.current || {}, backendKeyMap, fileQuestionMap, questionIndex), form_schema: buildVoiceFormContract(formSchema, questionIndex, backendKeyMap, fileQuestionMap, questionSectionMap), } : result; noteToolResult('getIntakeState', { action: 'state' }); return withVisibleState; }, proposeAnswer: async (raw) => { const params = normalizeToolParams(raw); noteToolCall('proposeAnswer', params); setPhase('saving'); const normalizedParams = params.question_key && !valueIsEmpty(params.answer_value) && normalizeBackendAnswer ? { ...params, answer_value: normalizeBackendAnswer(params.question_key, params.answer_value) } : params; if (normalizedParams.question_key && validateBackendAnswer) { const validation = validateBackendAnswer(normalizedParams.question_key, normalizedParams.answer_value); if (!validation.ok) { const result = { ok: false, action: 'invalid_format', message: validation.message || 'That answer does not match the expected field format.', question_key: normalizedParams.question_key, }; noteToolResult('proposeAnswer', { action: result.action, question_key: normalizedParams.question_key }); setPhase('listening'); return result; } } const result = await proposeVoiceAnswer(normalizedParams); applyToolState(result, params.question_key); setPhase(result.action === 'needs_confirmation' ? 'confirm' : 'listening'); noteToolResult('proposeAnswer', { action: result?.action, question_key: params.question_key }); return result; }, confirmOverwrite: async (raw) => { const params = normalizeToolParams(raw); noteToolCall('confirmOverwrite', params); return confirmPendingOverwrite(params, 'replace'); }, markUnresolved: async (raw) => { const params = normalizeToolParams(raw); noteToolCall('markUnresolved', params); const result = await markVoiceUnresolved(params); applyToolState(result, params.question_key, 'unresolved'); noteToolResult('markUnresolved', { action: result?.action, question_key: params.question_key }); return result; }, focusQuestion: async (raw) => { noteToolCall('focusQuestion', normalizeToolParams(raw)); const result = await focusQuestion(raw); noteToolResult('focusQuestion', { action: 'focused', question_key: normalizeToolParams(raw).question_key }); return result; }, attachArtifact: async (raw) => { const params = normalizeToolParams(raw); noteToolCall('attachArtifact', params); const result = await attachArtifactLink({ file_category: params.file_category || VOICE_DEFAULT_DOCUMENT_FILE_CATEGORY, url: params.url || params.file_id_or_url, label: params.label, question_key: params.question_key, }); addActivity({ key: params.question_key || params.file_category, label: 'Artifact attached', value: result.original_filename, kind: 'attached' }); noteToolResult('attachArtifact', { action: 'attached', question_key: params.question_key }); return result; }, showEducationCard: async (raw) => { noteToolCall('showEducationCard', normalizeToolParams(raw)); const { topic, body } = normalizeToolParams(raw); setEducation({ title: topic || 'NeevCloud intake', body: body || '' }); setPhase('education'); noteToolResult('showEducationCard', { action: 'shown' }); return { ok: true }; }, prepareReview: async () => { noteToolCall('prepareReview'); const result = await prepareVoiceReview(); if (result?.voice_state) syncVoiceState(result.voice_state); if (result.action === 'ready_for_review') setPhase('reviewing'); noteToolResult('prepareReview', { action: result?.action }); return result; }, }), [ addActivity, applyToolState, attachArtifactLink, backendKeyMap, confirmPendingOverwrite, fileQuestionMap, focusQuestion, formSchema, markVoiceUnresolved, noteToolCall, noteToolResult, normalizeBackendAnswer, validateBackendAnswer, prepareVoiceReview, proposeVoiceAnswer, questionIndex, questionSectionMap, refreshState, saveAnswerAndAdvance, startNewIntake, syncVoiceState, ]); const start = async () => { setRunning(true); setPhase('thinking'); manualDisconnectRef.current = false; audioChunkCountRef.current = 0; lastAgentAudioAtRef.current = 0; lastAgentBarsRef.current = barsFromFrequencyData(null); setConnectionError(''); try { if (!questionIndex?.schemaVersion) { throw new Error('The canonical form schema has not loaded yet. Please try the voice survey again in a moment.'); } const startupState = await refreshState(); const firstMessage = chooseFirstMessageForVoiceSession( startupState, answersRef.current || {}, backendKeyMap, fileQuestionMap, questionIndex, ); logToolEvent('voice_first_message_selected', firstMessage); pushTranscript('agent', firstMessage.message); if (!navigator.mediaDevices?.getUserMedia) { throw new Error('This browser does not expose microphone access.'); } const permissionStream = await navigator.mediaDevices.getUserMedia({ audio: true }); permissionStream.getTracks().forEach((track) => track.stop()); const token = await getVoiceToken(); const Conversation = window.NeevElevenLabsClient?.Conversation; if (!Conversation?.startSession) throw new Error('Local ElevenLabs browser SDK bundle did not load.'); const sessionAuth = token.signed_url ? { signedUrl: token.signed_url, connectionType: token.connection_type || 'websocket' } : { conversationToken: token.conversation_token, connectionType: token.connection_type || 'webrtc' }; if (!sessionAuth.signedUrl && !sessionAuth.conversationToken) { throw new Error('Backend did not return an ElevenLabs signed URL or conversation token.'); } const conversation = await Conversation.startSession({ ...sessionAuth, clientTools, overrides: { agent: { firstMessage: firstMessage.message, }, }, onConnect: () => { setPhase('listening'); addActivity({ key: 'voice', label: 'Voice session', value: 'Connected', kind: 'connected' }); }, onDisconnect: (details) => { const wasManual = manualDisconnectRef.current || details?.reason === 'user'; setPhase(wasManual ? 'idle' : 'disconnected'); setRunning(false); if (!wasManual) setConnectionError(describeVoiceDisconnect(details)); addActivity({ key: 'voice', label: 'Voice session', value: wasManual ? 'Ended. Draft preserved.' : `${describeVoiceDisconnect(details)} Draft preserved.`, kind: wasManual ? 'disconnected' : 'error', }); }, onError: (error) => { setPhase('disconnected'); const message = describeVoiceError(error); setConnectionError(message); addActivity({ key: 'voice', label: 'Voice error', value: message, kind: 'error' }); }, onStatusChange: ({ status }) => { if (status === 'connected') setPhase('listening'); if (status === 'connecting') setPhase('thinking'); if (status === 'disconnected') setPhase('disconnected'); }, onModeChange: ({ mode }) => { if (mode === 'speaking' || mode === 'listening') setPhase(mode); }, onAudio: (audioBase64) => { audioChunkCountRef.current += 1; if (latencyRef.current.waitingForAgentAudio) { latencyRef.current.waitingForAgentAudio = false; logLatency('first_agent_audio_after_tool_result'); } const bars = barsFromPcmBase64(audioBase64); lastAgentAudioAtRef.current = Date.now(); lastAgentBarsRef.current = bars; setAudioBars(bars); setPhase('speaking'); if (audioChunkCountRef.current === 1) { addActivity({ key: 'voice-audio', label: 'Audio output', value: 'Receiving agent speech', kind: 'connected' }); } unlockConversationAudio(conversationRef.current).catch(() => {}); }, onMessage: (message) => { const text = message?.message || message?.text || ''; if (text) { const who = message?.source === 'user' ? 'user' : 'agent'; if (who === 'user') { const now = Math.round(performance?.now?.() || Date.now()); latencyRef.current.turnId += 1; latencyRef.current.userTranscriptAt = now; latencyRef.current.firstToolSeenForTurn = false; latencyRef.current.toolResultAt = null; latencyRef.current.waitingForAgentText = false; latencyRef.current.waitingForAgentAudio = false; logLatency('user_transcript_received', { text_length: text.length }); } else if (latencyRef.current.waitingForAgentText) { latencyRef.current.waitingForAgentText = false; logLatency('first_agent_text_after_tool_result', { text_length: text.length }); } pushTranscript(who, text); } }, }); conversationRef.current = conversation; await unlockConversationAudio(conversation); const conversationId = conversation?.conversationId || conversation?.id || conversation?.getId?.(); if (conversationId) await registerVoiceConversation(conversationId, { source: 'updated_browser_panel' }); await refreshState(); } catch (err) { setPhase('disconnected'); setRunning(false); const message = describeVoiceError(err); setConnectionError(message); addActivity({ key: 'voice', label: 'Voice unavailable', value: `${message}. Text mode remains available.`, kind: 'error' }); } }; const pause = async () => { manualDisconnectRef.current = true; try { await conversationRef.current?.endSession?.(); } finally { conversationRef.current = null; setPhase('idle'); setRunning(false); } }; const end = async () => { await pause(); const result = await prepareVoiceReview?.().catch(() => null); if (result?.voice_state) syncVoiceState(result.voice_state); setPhase(result?.action === 'ready_for_review' ? 'reviewing' : 'idle'); }; const addUpload = async (file, kind = VOICE_DEFAULT_DOCUMENT_FILE_CATEGORY) => { const upload = { name: file.name, size: file.size, file, kind, pendingClassify: false, ts: Date.now() }; setUploads((u) => [upload, ...u].slice(0, 6)); await classifyUpload(0, kind, upload); }; const classifyUpload = async (idx, kind, existingUpload = null) => { const upload = existingUpload || uploads[idx]; if (!upload?.file || !uploadArtifact) return; setPhase('saving'); const uploaded = await uploadArtifact(upload.file, kind); const questionKey = `files.${kind}`; setUploads((u) => u.map((x, i) => i === idx ? { ...x, kind, pendingClassify: false, uploaded } : x)); addActivity({ key: questionKey, label: 'Artifact attached', value: uploaded.original_filename, ts: Date.now(), kind: 'attached' }); pushTranscript('agent', `Uploaded ${uploaded.original_filename}. I will classify it in the backend.`); setPhase(running ? 'listening' : 'idle'); }; const askEducation = (title, body) => { setEducation({ title, body }); setPhase('education'); }; const closeEducation = () => { setEducation(null); setPhase(running ? 'listening' : 'idle'); }; const sendText = async (text) => { const sent = text.trim(); if (!sent) return; pushTranscript('user', sent); if (conversationRef.current?.sendUserMessage) { await conversationRef.current.sendUserMessage(sent); return; } if (/^https?:\/\//i.test(sent)) { const result = await attachArtifactLink?.({ url: sent, file_category: VOICE_DEFAULT_DOCUMENT_FILE_CATEGORY, label: sent }); addActivity({ key: 'files.site_documents', label: 'Link attached', value: result?.original_filename || sent, kind: 'attached' }); return; } const explicit = sent.match(/^([a-z_]+(?:\.[a-z0-9_]+)+)\s*:\s*(.+)$/i); const questionKey = explicit?.[1] || voiceState?.missing_required_question_keys?.[0]; const answerValue = explicit?.[2] || sent; if (!questionKey) { pushTranscript('agent', 'I need one field at a time. I will note that down, and someone from NeevCloud can get back to you.'); return; } await aiSave(questionKey, answerValue, labelForQuestion(questionKey, questionIndex, backendKeyMap), { evidence: sent, source: 'text_input' }); }; const value = { phase, setPhase, running, start, pause, end, currentIdx, setCurrentIdx, questionFlow, transcript, pushTranscript, activity, setActivity, pendingConfirm, setPendingConfirm: setPendingConfirmState, followUps, setFollowUps, needsConfirm, setNeedsConfirm, highlight, flashField, education, askEducation, closeEducation, uploads, addUpload, classifyUpload, aiSave, acceptOverwrite, keepOriginal, markFollowUp, registerField, voiceState, refreshState, connectionError, sendText, audioBars, backendKeyMap, fileQuestionMap, questionSectionMap, }; return {children}; } // ========================================================================= // VOICE PANEL // ========================================================================= function VoicePanel({ completion, requiredCount, requiredFilled, onCollapse, collapsed }) { const v = useVoice(); if (collapsed) return ; return ( ); } function VoicePanelCollapsed({ onExpand }) { const v = useVoice(); return ( ); } function VPHeader({ onCollapse }) { const v = useVoice(); return (
NeevCloud AI Intake Agent
{phaseLabel(v.phase)}{v.running && ' · in progress'}
); } function VPState() { const v = useVoice(); return (
{phaseLabel(v.phase)}
{v.connectionError || phaseSub(v.phase)}
); } function phaseLabel(p) { return ({ idle: 'Ready', ready: 'Ready', listening: 'Listening', speaking: 'Speaking', thinking: 'Thinking', saving: 'Saving answer', confirm: 'Needs confirmation', disconnected: 'Disconnected', reviewing: 'Ready for review', education: 'Explaining', })[p] || 'Ready'; } function phaseSub(p) { return ({ idle: 'Tap start when you\'re ready.', ready: 'Tap start when you\'re ready.', listening: 'Speak naturally — I\'m taking notes.', speaking: 'Asking the next question.', thinking: 'Working out where this goes.', saving: 'Writing this into the form.', confirm: 'You said something different from what\'s already saved.', disconnected: 'Audio dropped — text input still works.', reviewing: 'All required fields captured. Review on the left.', education: 'Quick explainer — back to questions in a moment.', })[p] || ''; } function Visualizer({ phase, bars }) { const values = bars?.length ? bars : barsFromFrequencyData(null); return ( ); } function VPCurrentQuestion() { const v = useVoice(); const q = v.questionFlow[v.currentIdx]; if (!q || v.phase === 'reviewing') return null; return (
Current question

{q.q}

Approximate is fine if exact figures aren't available. You can edit this on the left before submitting.
Next {(v.questionFlow[v.currentIdx + 1] || {}).q || 'Wrapping up — review and submit.'}
); } function VPConfirm() { const v = useVoice(); const c = v.pendingConfirm; if (!c) return null; return (
Confirm overwrite
You already had an answer for {c.label}.
Existing{String(c.oldValue)}
New from voice{String(c.newValue)}

Say yes to replace it, say keep existing, or use the buttons here.

); } function VPEducation() { const v = useVoice(); return (
Quick explainer

{v.education.title}

{v.education.body}

); } function VPProgress({ completion, requiredCount, requiredFilled }) { const v = useVoice(); const ready = requiredFilled === requiredCount; return (
Intake progress
Required fields {requiredFilled} / {requiredCount}
Needs confirmation{v.needsConfirm.length}
Unresolved{v.followUps.length}
Ready for review{ready ? 'Yes' : 'No'}
); } function VPTranscript() { const v = useVoice(); const [open, setOpen] = useState(false); if (v.transcript.length === 0) return null; return (
{open && ( )}
); } function VPActivity() { const v = useVoice(); const [open, setOpen] = useState(false); if (v.activity.length === 0) return null; return (
{open && ( )}
); } function VPNeedsConfirm() { const v = useVoice(); return (
Needs your confirmation
); } function VPFollowUps() { const v = useVoice(); return (
Marked for follow-up

Someone from NeevCloud can follow up on these — I won't block submission on them.

); } function VPInput() { const v = useVoice(); const [text, setText] = useState(''); const submit = async () => { if (!text.trim()) return; await v.sendText(text); setText(''); }; return (
Type or paste here
setText(e.target.value)} onKeyDown={(e) => e.key === 'Enter' && submit()} />
); } function VPUploads() { const v = useVoice(); const [drag, setDrag] = useState(false); const inputRef = useRef(null); const onDrop = (e) => { e.preventDefault(); setDrag(false); const f = e.dataTransfer.files?.[0]; if (f) v.addUpload(f); }; const onPick = () => inputRef.current?.click(); return (
Uploads
{ e.preventDefault(); setDrag(true); }} onDragLeave={() => setDrag(false)} onDrop={onDrop} > ↓ Drag a file here, or PDF · DWG · PNG · JPG · ZIP · 50 MB max
{ const f = e.target.files?.[0]; if (f) v.addUpload(f); e.target.value = ''; }} /> {v.uploads.length > 0 && ( )}
); } function VPControls() { const v = useVoice(); if (v.phase === 'reviewing') { return ( ); } if (v.phase === 'disconnected') { return ( ); } if (!v.running) { return ( ); } return ( ); } // Expose to global Object.assign(window, { VoiceProvider, VoicePanel, useVoice }); }