refactor chat delivery to graphql + hatchet services

telegram_worker/.dockerignore

@@ -0,0 +1,5 @@
+node_modules
+npm-debug.log
+.git
+.gitignore
+.DS_Store

telegram_worker/Dockerfile

@@ -0,0 +1,13 @@
+FROM node:22-alpine
+
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm ci
+
+COPY src ./src
+COPY tsconfig.json ./tsconfig.json
+
+ENV NODE_ENV=production
+
+CMD ["npm", "run", "start"]

telegram_worker/README.md

@@ -0,0 +1,30 @@
+# telegram_worker
+
+Hatchet worker для Telegram-цепочки.
+
+## Назначение
+
+- выполняет `process-telegram-inbound`:
+  - забирает нормализованный inbound envelope;
+  - пишет событие в `backend` через GraphQL mutation `ingestTelegramInbound`.
+- выполняет `process-telegram-outbound`:
+  - отправляет сообщение через `telegram_backend` mutation `sendTelegramMessage`;
+  - репортит статус в `backend` mutation `reportTelegramOutbound`.
+- ретраи/бекофф выполняются через Hatchet.
+
+## Переменные окружения
+
+- `BACKEND_GRAPHQL_URL` (required)
+- `BACKEND_GRAPHQL_SHARED_SECRET` (optional)
+- `BACKEND_REPORT_RETRIES` (default: `6`)
+- `TELEGRAM_BACKEND_GRAPHQL_URL` (required)
+- `TELEGRAM_BACKEND_GRAPHQL_SHARED_SECRET` (optional)
+- `HATCHET_CLIENT_TOKEN` (required)
+- `HATCHET_CLIENT_TLS_STRATEGY` (для self-host без TLS: `none`)
+- `HATCHET_CLIENT_HOST_PORT` (например, `hatchet-engine:7070`)
+- `HATCHET_CLIENT_API_URL` (URL Hatchet API)
+
+## Скрипты
+
+- `npm run start` — запуск Hatchet worker.
+- `npm run typecheck` — проверка TypeScript.

telegram_worker/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "crm-telegram-worker",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "start": "tsx src/hatchet/worker.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@hatchet-dev/typescript-sdk": "^1.15.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.13.9",
+    "tsx": "^4.20.5",
+    "typescript": "^5.9.2"
+  }
+}

telegram_worker/src/hatchet/client.ts

@@ -0,0 +1,3 @@
+import { HatchetClient } from "@hatchet-dev/typescript-sdk/v1";
+
+export const hatchet = HatchetClient.init();

telegram_worker/src/hatchet/tasks.ts

@@ -0,0 +1,347 @@
+import { NonRetryableError } from "@hatchet-dev/typescript-sdk/v1/task";
+import { hatchet } from "./client";
+
+type OmniInboundEnvelopeV1 = {
+  version: number;
+  idempotencyKey: string;
+  provider: string;
+  channel: "TELEGRAM" | "WHATSAPP" | "INSTAGRAM" | "PHONE" | "EMAIL" | "INTERNAL";
+  direction: "IN" | "OUT";
+  providerEventId: string;
+  providerMessageId: string | null;
+  eventType: string;
+  occurredAt: string;
+  receivedAt: string;
+  payloadRaw: unknown;
+  payloadNormalized: Record<string, unknown>;
+};
+
+type TelegramOutboundTaskInput = {
+  omniMessageId: string;
+  chatId: string;
+  text: string;
+  businessConnectionId?: string | null;
+};
+
+type GraphqlResponse<T> = {
+  data?: T;
+  errors?: Array<{ message?: string }>;
+};
+
+type GraphqlRequest = {
+  query: string;
+  variables?: Record<string, unknown>;
+  operationName?: string;
+};
+
+function asString(value: unknown) {
+  if (typeof value !== "string") return null;
+  const v = value.trim();
+  return v || null;
+}
+
+function requiredEnv(name: string) {
+  const value = asString(process.env[name]);
+  if (!value) {
+    throw new Error(`${name} is required`);
+  }
+  return value;
+}
+
+function safeJsonString(value: unknown) {
+  try {
+    return JSON.stringify(value ?? null);
+  } catch {
+    return JSON.stringify({ unserializable: true });
+  }
+}
+
+function compactError(error: unknown) {
+  if (!error) return "unknown_error";
+  if (typeof error === "string") return error;
+  const anyErr = error as { message?: unknown; stack?: unknown };
+  return String(anyErr.message ?? anyErr.stack ?? "unknown_error");
+}
+
+async function sleep(ms: number) {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function callGraphql<T>(
+  url: string,
+  body: GraphqlRequest,
+  secret: string | null,
+): Promise<T> {
+  const headers: Record<string, string> = {
+    "content-type": "application/json",
+  };
+
+  if (secret) {
+    headers["x-graphql-secret"] = secret;
+  }
+
+  const response = await fetch(url, {
+    method: "POST",
+    headers,
+    body: JSON.stringify(body),
+  });
+
+  const payload = (await response.json()) as GraphqlResponse<T>;
+  if (!response.ok || payload.errors?.length) {
+    const message =
+      payload.errors?.map((error) => error.message).filter(Boolean).join("; ") || `HTTP ${response.status}`;
+    throw new Error(message);
+  }
+
+  if (!payload.data) {
+    throw new Error("graphql_data_missing");
+  }
+
+  return payload.data;
+}
+
+async function reportToBackend(input: {
+  omniMessageId: string;
+  status: "SENT" | "FAILED";
+  providerMessageId?: string | null;
+  error?: string | null;
+  responseJson?: string | null;
+}) {
+  type ReportResult = {
+    reportTelegramOutbound: {
+      ok: boolean;
+      message: string;
+    };
+  };
+
+  const url = requiredEnv("BACKEND_GRAPHQL_URL");
+  const secret = asString(process.env.BACKEND_GRAPHQL_SHARED_SECRET);
+
+  const query = `mutation ReportTelegramOutbound($input: TelegramOutboundReportInput!) {
+    reportTelegramOutbound(input: $input) {
+      ok
+      message
+    }
+  }`;
+
+  const data = await callGraphql<ReportResult>(
+    url,
+    {
+      operationName: "ReportTelegramOutbound",
+      query,
+      variables: {
+        input: {
+          omniMessageId: input.omniMessageId,
+          status: input.status,
+          providerMessageId: input.providerMessageId ?? null,
+          error: input.error ?? null,
+          responseJson: input.responseJson ?? null,
+        },
+      },
+    },
+    secret,
+  );
+
+  if (!data.reportTelegramOutbound.ok) {
+    throw new Error(data.reportTelegramOutbound.message || "backend_report_failed");
+  }
+}
+
+async function reportToBackendWithRetry(
+  input: Parameters<typeof reportToBackend>[0],
+  attempts: number,
+) {
+  let lastError: unknown = null;
+
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    try {
+      await reportToBackend(input);
+      return;
+    } catch (error) {
+      lastError = error;
+      if (attempt < attempts) {
+        const delayMs = Math.min(30000, 250 * 2 ** attempt);
+        await sleep(delayMs);
+      }
+    }
+  }
+
+  throw lastError instanceof Error ? lastError : new Error(compactError(lastError));
+}
+
+async function ingestInboundToBackend(input: OmniInboundEnvelopeV1) {
+  type IngestResult = {
+    ingestTelegramInbound: {
+      ok: boolean;
+      message: string;
+      omniMessageId?: string | null;
+    };
+  };
+
+  const url = requiredEnv("BACKEND_GRAPHQL_URL");
+  const secret = asString(process.env.BACKEND_GRAPHQL_SHARED_SECRET);
+
+  const query = `mutation IngestTelegramInbound($input: TelegramInboundInput!) {
+    ingestTelegramInbound(input: $input) {
+      ok
+      message
+      omniMessageId
+    }
+  }`;
+
+  const data = await callGraphql<IngestResult>(
+    url,
+    {
+      operationName: "IngestTelegramInbound",
+      query,
+      variables: {
+        input: {
+          version: input.version,
+          idempotencyKey: input.idempotencyKey,
+          provider: input.provider,
+          channel: input.channel,
+          direction: input.direction,
+          providerEventId: input.providerEventId,
+          providerMessageId: input.providerMessageId,
+          eventType: input.eventType,
+          occurredAt: input.occurredAt,
+          receivedAt: input.receivedAt,
+          payloadRawJson: safeJsonString(input.payloadRaw),
+          payloadNormalizedJson: safeJsonString(input.payloadNormalized),
+        },
+      },
+    },
+    secret,
+  );
+
+  if (!data.ingestTelegramInbound.ok) {
+    throw new Error(data.ingestTelegramInbound.message || "backend_ingest_failed");
+  }
+}
+
+async function sendViaTelegramBackend(input: TelegramOutboundTaskInput) {
+  type SendResult = {
+    sendTelegramMessage: {
+      ok: boolean;
+      message: string;
+      providerMessageId?: string | null;
+      responseJson?: string | null;
+    };
+  };
+
+  const url = requiredEnv("TELEGRAM_BACKEND_GRAPHQL_URL");
+  const secret = asString(process.env.TELEGRAM_BACKEND_GRAPHQL_SHARED_SECRET);
+
+  const query = `mutation SendTelegramMessage($input: TelegramSendMessageInput!) {
+    sendTelegramMessage(input: $input) {
+      ok
+      message
+      providerMessageId
+      responseJson
+    }
+  }`;
+
+  const data = await callGraphql<SendResult>(
+    url,
+    {
+      operationName: "SendTelegramMessage",
+      query,
+      variables: {
+        input: {
+          chatId: input.chatId,
+          text: input.text,
+          businessConnectionId: input.businessConnectionId ?? null,
+        },
+      },
+    },
+    secret,
+  );
+
+  if (!data.sendTelegramMessage.ok) {
+    throw new Error(data.sendTelegramMessage.message || "telegram_send_failed");
+  }
+
+  return {
+    providerMessageId: data.sendTelegramMessage.providerMessageId ?? null,
+    responseJson: data.sendTelegramMessage.responseJson ?? null,
+  };
+}
+
+export const processTelegramInbound = hatchet.task({
+  name: "process-telegram-inbound",
+  retries: 12,
+  backoff: { factor: 2, maxSeconds: 60 },
+  fn: async (input: any, ctx) => {
+    const envelope = input as OmniInboundEnvelopeV1;
+    await ingestInboundToBackend(envelope);
+
+    await ctx.logger.info("telegram inbound processed", {
+      providerEventId: envelope.providerEventId,
+      idempotencyKey: envelope.idempotencyKey,
+      eventType: envelope.eventType,
+    });
+
+    return { ok: true };
+  },
+});
+
+export const processTelegramOutbound = hatchet.task({
+  name: "process-telegram-outbound",
+  retries: 12,
+  backoff: { factor: 2, maxSeconds: 60 },
+  fn: async (input: any, ctx) => {
+    const payload = input as TelegramOutboundTaskInput;
+    try {
+      const sent = await sendViaTelegramBackend(payload);
+
+      const reportAttempts = Math.max(1, Number(process.env.BACKEND_REPORT_RETRIES || 6));
+      try {
+        await reportToBackendWithRetry(
+          {
+            omniMessageId: payload.omniMessageId,
+            status: "SENT",
+            providerMessageId: sent.providerMessageId,
+            responseJson: sent.responseJson,
+          },
+          reportAttempts,
+        );
+      } catch (reportError) {
+        await ctx.logger.error("telegram sent but backend status report failed", {
+          error: reportError instanceof Error ? reportError : new Error(compactError(reportError)),
+          extra: {
+            omniMessageId: payload.omniMessageId,
+            providerMessageId: sent.providerMessageId,
+          },
+        });
+
+        throw new NonRetryableError(
+          `telegram sent but backend report failed: ${compactError(reportError)}`,
+        );
+      }
+
+      await ctx.logger.info("telegram outbound processed", {
+        omniMessageId: payload.omniMessageId,
+        providerMessageId: sent.providerMessageId,
+      });
+
+      return {
+        ok: true,
+        providerMessageId: sent.providerMessageId,
+      };
+    } catch (error) {
+      const errorMessage = compactError(error);
+
+      try {
+        await reportToBackend({
+          omniMessageId: payload.omniMessageId,
+          status: "FAILED",
+          error: errorMessage,
+        });
+      } catch {
+        // Ignore report failure here: the task itself will retry.
+      }
+
+      throw error;
+    }
+  },
+});

telegram_worker/src/hatchet/worker.ts

@@ -0,0 +1,22 @@
+import { hatchet } from "./client";
+import { processTelegramInbound, processTelegramOutbound } from "./tasks";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+async function main() {
+  const worker = await hatchet.worker("telegram-worker", {
+    workflows: [processTelegramInbound, processTelegramOutbound],
+  });
+
+  await worker.start();
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+
+if (isMain) {
+  main().catch((error) => {
+    const message = error instanceof Error ? error.stack || error.message : String(error);
+    console.error(`[telegram_worker/hatchet] worker failed: ${message}`);
+    process.exitCode = 1;
+  });
+}

telegram_worker/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "strict": true,
+    "skipLibCheck": true,
+    "esModuleInterop": true,
+    "types": ["node"],
+    "resolveJsonModule": true,
+    "verbatimModuleSyntax": true
+  },
+  "include": ["src/**/*.ts"]
+}