From 94c01516ba95fd7b4a6a4e3d22dde82a2ae24c61 Mon Sep 17 00:00:00 2001 From: Ruslan Bakiev <572431+veikab@users.noreply.github.com> Date: Mon, 23 Feb 2026 16:02:57 +0700 Subject: [PATCH] precompute call waveforms and stop list-time audio loading --- .../components/workspace/CrmWorkspaceApp.vue | 22 +++-- .../graphql/operations/communications.graphql | 1 + .../operations/get-client-timeline.graphql | 1 + .../migration.sql | 3 + frontend/prisma/schema.prisma | 1 + frontend/server/graphql/schema.ts | 7 ++ omni_chat/prisma/schema.prisma | 1 + omni_chat/src/worker.ts | 98 +++++++++++++++++++ omni_outbound/prisma/schema.prisma | 1 + 9 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 frontend/prisma/migrations/4_add_contact_message_waveform_json/migration.sql diff --git a/frontend/app/components/workspace/CrmWorkspaceApp.vue b/frontend/app/components/workspace/CrmWorkspaceApp.vue index 3db145e..4300b65 100644 --- a/frontend/app/components/workspace/CrmWorkspaceApp.vue +++ b/frontend/app/components/workspace/CrmWorkspaceApp.vue @@ -103,6 +103,7 @@ type CommItem = { text: string; audioUrl?: string; duration?: string; + waveform?: number[]; transcript?: string[]; deliveryStatus?: "PENDING" | "SENT" | "DELIVERED" | "READ" | "FAILED" | string | null; }; @@ -1262,6 +1263,19 @@ function parseDurationToSeconds(raw?: string) { } function buildCallWavePeaks(item: CommItem, size = 320) { + const stored = Array.isArray(item.waveform) + ? item.waveform.map((value) => Number(value)).filter((value) => Number.isFinite(value) && value > 0) + : []; + if (stored.length) { + const sampled = new Float32Array(size); + for (let i = 0; i < size; i += 1) { + const t = size <= 1 ? 0 : i / (size - 1); + const idx = Math.min(stored.length - 1, Math.round(t * (stored.length - 1))); + sampled[i] = Math.max(0.05, Math.min(1, stored[idx] ?? 0.05)); + } + return sampled; + } + const source = `${item.text} ${(item.transcript ?? []).join(" ")}`.trim() || item.contact; let seed = 0; for (let i = 0; i < source.length; i += 1) { @@ -1295,7 +1309,6 @@ async function ensureCommCallWave(itemId: string) { const callItem = visibleThreadItems.value.find((item) => item.id === itemId && item.kind === "call"); if (!callItem) return; - const audioUrl = getCallAudioUrl(callItem); const { WaveSurfer } = await loadWaveSurferModules(); const durationSeconds = @@ -1314,12 +1327,7 @@ async function ensureCommCallWave(itemId: string) { barWidth: 0, }); - try { - if (!audioUrl) throw new Error("missing_audio_url"); - await ws.load(audioUrl); - } catch { - await ws.load("", [peaks], durationSeconds); - } + await ws.load("", [peaks], durationSeconds); commCallWaveSurfers.set(itemId, ws); } diff --git a/frontend/graphql/operations/communications.graphql b/frontend/graphql/operations/communications.graphql index 5661b64..aeae1f9 100644 --- a/frontend/graphql/operations/communications.graphql +++ b/frontend/graphql/operations/communications.graphql @@ -13,6 +13,7 @@ query CommunicationsQuery { text audioUrl duration + waveform transcript deliveryStatus } diff --git a/frontend/graphql/operations/get-client-timeline.graphql b/frontend/graphql/operations/get-client-timeline.graphql index 359b306..08181b7 100644 --- a/frontend/graphql/operations/get-client-timeline.graphql +++ b/frontend/graphql/operations/get-client-timeline.graphql @@ -19,6 +19,7 @@ query GetClientTimelineQuery($contactId: ID!, $limit: Int) { text audioUrl duration + waveform transcript deliveryStatus } diff --git a/frontend/prisma/migrations/4_add_contact_message_waveform_json/migration.sql b/frontend/prisma/migrations/4_add_contact_message_waveform_json/migration.sql new file mode 100644 index 0000000..f29577c --- /dev/null +++ b/frontend/prisma/migrations/4_add_contact_message_waveform_json/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "ContactMessage" ADD COLUMN "waveformJson" JSONB; + diff --git a/frontend/prisma/schema.prisma b/frontend/prisma/schema.prisma index 7c0a781..7e54e06 100644 --- a/frontend/prisma/schema.prisma +++ b/frontend/prisma/schema.prisma @@ -166,6 +166,7 @@ model ContactMessage { content String audioUrl String? durationSec Int? + waveformJson Json? transcriptJson Json? occurredAt DateTime @default(now()) createdAt DateTime @default(now()) diff --git a/frontend/server/graphql/schema.ts b/frontend/server/graphql/schema.ts index 29e7d80..c3a17de 100644 --- a/frontend/server/graphql/schema.ts +++ b/frontend/server/graphql/schema.ts @@ -633,6 +633,9 @@ async function getCommunications(auth: AuthContext | null) { text: m.content, audioUrl: resolveContactMessageAudioUrl(m), duration: m.durationSec ? new Date(m.durationSec * 1000).toISOString().slice(14, 19) : "", + waveform: Array.isArray(m.waveformJson) + ? m.waveformJson.map((value) => Number(value)).filter((value) => Number.isFinite(value)) + : [], transcript: Array.isArray(m.transcriptJson) ? ((m.transcriptJson as any) as string[]) : [], deliveryStatus: resolveDeliveryStatus(m), })); @@ -953,6 +956,9 @@ async function getClientTimeline(auth: AuthContext | null, contactIdInput: strin text: m.content, audioUrl: resolveContactMessageAudioUrl(m), duration: m.durationSec ? new Date(m.durationSec * 1000).toISOString().slice(14, 19) : "", + waveform: Array.isArray(m.waveformJson) + ? m.waveformJson.map((value) => Number(value)).filter((value) => Number.isFinite(value)) + : [], transcript: Array.isArray(m.transcriptJson) ? ((m.transcriptJson as any) as string[]) : [], deliveryStatus: resolveDeliveryStatus(m), }, @@ -2014,6 +2020,7 @@ export const crmGraphqlSchema = buildSchema(` text: String! audioUrl: String! duration: String! + waveform: [Float!]! transcript: [String!]! deliveryStatus: String } diff --git a/omni_chat/prisma/schema.prisma b/omni_chat/prisma/schema.prisma index bf11af7..83ad487 100644 --- a/omni_chat/prisma/schema.prisma +++ b/omni_chat/prisma/schema.prisma @@ -158,6 +158,7 @@ model ContactMessage { content String audioUrl String? durationSec Int? + waveformJson Json? transcriptJson Json? occurredAt DateTime @default(now()) createdAt DateTime @default(now()) diff --git a/omni_chat/src/worker.ts b/omni_chat/src/worker.ts index c527a72..6769914 100644 --- a/omni_chat/src/worker.ts +++ b/omni_chat/src/worker.ts @@ -29,6 +29,7 @@ type OmniInboundEnvelopeV1 = { export const RECEIVER_FLOW_QUEUE_NAME = (process.env.RECEIVER_FLOW_QUEUE_NAME || "receiver.flow").trim(); const TELEGRAM_PLACEHOLDER_PREFIX = "Telegram "; const TELEGRAM_AUDIO_FILE_MARKER = "tg-file:"; +const TELEGRAM_WAVE_BINS = 96; function redisConnectionFromEnv(): ConnectionOptions { const raw = (process.env.REDIS_URL || "redis://localhost:6379").trim(); @@ -87,6 +88,101 @@ function fallbackTextFromMedia(media: TelegramInboundMedia) { return "[audio]"; } +function buildFallbackWaveform(seedText: string, bins = TELEGRAM_WAVE_BINS) { + let seed = 0; + for (let i = 0; i < seedText.length; i += 1) { + seed = (seed * 33 + seedText.charCodeAt(i)) >>> 0; + } + + const random = () => { + seed = (seed * 1664525 + 1013904223) >>> 0; + return seed / 0xffffffff; + }; + + const out: number[] = []; + let smooth = 0; + for (let i = 0; i < bins; i += 1) { + const t = i / Math.max(1, bins - 1); + const burst = Math.max(0, Math.sin(t * Math.PI * (2 + (seedText.length % 5)))); + const noise = (random() * 2 - 1) * 0.6; + smooth = smooth * 0.72 + noise * 0.28; + const value = Math.max(0.06, Math.min(1, 0.12 + Math.abs(smooth) * 0.42 + burst * 0.4)); + out.push(Number(value.toFixed(4))); + } + return out; +} + +function buildWaveformFromBytes(bytes: Uint8Array, bins = TELEGRAM_WAVE_BINS) { + if (!bytes.length) return []; + const bucketSize = Math.max(1, Math.ceil(bytes.length / bins)); + const raw = new Array(bins).fill(0); + + for (let i = 0; i < bins; i += 1) { + const start = i * bucketSize; + const end = Math.min(bytes.length, start + bucketSize); + if (start >= end) continue; + + let energy = 0; + for (let j = start; j < end; j += 1) { + energy += Math.abs(bytes[j] - 128) / 128; + } + raw[i] = energy / (end - start); + } + + const smooth: number[] = []; + let prev = 0; + for (const value of raw) { + prev = prev * 0.78 + value * 0.22; + smooth.push(prev); + } + + const maxValue = Math.max(...smooth, 0); + if (maxValue <= 0) return []; + + return smooth.map((value) => { + const normalized = value / maxValue; + const mapped = Math.max(0.06, Math.min(1, normalized * 0.9 + 0.06)); + return Number(mapped.toFixed(4)); + }); +} + +async function fetchTelegramFileBytes(fileId: string) { + const token = String(process.env.TELEGRAM_BOT_TOKEN ?? "").trim(); + if (!token) return null; + + const base = String(process.env.TELEGRAM_API_BASE ?? "https://api.telegram.org").replace(/\/+$/, ""); + + const metaRes = await fetch(`${base}/bot${token}/getFile`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ file_id: fileId }), + }); + const metaJson = (await metaRes.json().catch(() => null)) as + | { ok?: boolean; result?: { file_path?: string } } + | null; + const filePath = String(metaJson?.result?.file_path ?? "").trim(); + if (!metaRes.ok || !metaJson?.ok || !filePath) return null; + + const fileRes = await fetch(`${base}/file/bot${token}/${filePath}`); + if (!fileRes.ok) return null; + return new Uint8Array(await fileRes.arrayBuffer()); +} + +async function resolveInboundWaveform(media: TelegramInboundMedia, text: string) { + const fallback = buildFallbackWaveform(`${media.fileId ?? "none"}:${media.durationSec ?? "0"}:${text}`); + const fileId = media.fileId; + if (!fileId) return fallback; + + try { + const bytes = await fetchTelegramFileBytes(fileId); + if (!bytes?.length) return fallback; + const fromFile = buildWaveformFromBytes(bytes); + return fromFile.length ? fromFile : fallback; + } catch { + return fallback; + } +} + function parseOccurredAt(input: string | null | undefined) { const d = new Date(String(input ?? "")); if (Number.isNaN(d.getTime())) return new Date(); @@ -380,6 +476,7 @@ async function ingestInbound(env: OmniInboundEnvelopeV1) { const isAudioLike = Boolean(media.fileId) && (media.kind === "voice" || media.kind === "audio" || media.kind === "video_note"); const contactMessageKind: "MESSAGE" | "CALL" = isAudioLike ? "CALL" : "MESSAGE"; const contactMessageAudioUrl = isAudioLike ? `${TELEGRAM_AUDIO_FILE_MARKER}${media.fileId}` : null; + const waveformPeaks = isAudioLike ? await resolveInboundWaveform(media, text) : null; const occurredAt = parseOccurredAt(env.occurredAt); const direction = safeDirection(env.direction); const contactProfile = buildContactProfile(n, externalContactId); @@ -482,6 +579,7 @@ async function ingestInbound(env: OmniInboundEnvelopeV1) { content: text, audioUrl: contactMessageAudioUrl, durationSec: media.durationSec, + ...(waveformPeaks ? { waveformJson: waveformPeaks as Prisma.InputJsonValue } : {}), occurredAt, }, }); diff --git a/omni_outbound/prisma/schema.prisma b/omni_outbound/prisma/schema.prisma index bf11af7..83ad487 100644 --- a/omni_outbound/prisma/schema.prisma +++ b/omni_outbound/prisma/schema.prisma @@ -158,6 +158,7 @@ model ContactMessage { content String audioUrl String? durationSec Int? + waveformJson Json? transcriptJson Json? occurredAt DateTime @default(now()) createdAt DateTime @default(now())