Add voice-based place recommendations

.env.example

@@ -1,5 +1,8 @@
 DATABASE_URL=postgresql://mapflow:mapflow@localhost:5432/mapflow
 HATCHET_CLIENT_TOKEN=
+DEEPGRAM_API_KEY=
+DEEPGRAM_MODEL=nova-3
+DEEPGRAM_LANGUAGE=ru
 OPENROUTER_API_KEY=
 OPENROUTER_MODEL=deepseek/deepseek-v4-flash
 HOST=0.0.0.0

src/ai/place-intent.ts

@@ -0,0 +1,110 @@
+import { config } from '../config.js';
+import { placeOntology, placeOntologyTags } from '../ontology/place-ontology.js';
+
+type OpenRouterResponse = {
+  choices?: Array<{
+    finish_reason?: string;
+    message?: {
+      content?: string;
+    };
+  }>;
+};
+
+export type PlaceIntent = {
+  tags: string[];
+  summary: string;
+};
+
+export async function classifyPlaceIntent(input: {
+  transcript: string;
+}): Promise<PlaceIntent> {
+  if (!config.openRouterApiKey) {
+    throw new Error('OPENROUTER_API_KEY is required for voice place filters.');
+  }
+
+  const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+    method: 'POST',
+    headers: {
+      authorization: `Bearer ${config.openRouterApiKey}`,
+      'content-type': 'application/json',
+      'http-referer': config.webAppUrl,
+      'x-title': 'MapFlow',
+    },
+    body: JSON.stringify({
+      model: config.openRouterModel,
+      provider: { require_parameters: true },
+      temperature: 0.1,
+      max_tokens: 900,
+      response_format: {
+        type: 'json_schema',
+        json_schema: {
+          name: 'place_voice_filter',
+          strict: true,
+          schema: {
+            type: 'object',
+            additionalProperties: false,
+            required: ['tags', 'summary'],
+            properties: {
+              tags: {
+                type: 'array',
+                maxItems: 8,
+                items: { type: 'string', enum: placeOntologyTags },
+              },
+              summary: { type: 'string' },
+            },
+          },
+        },
+      },
+      messages: [
+        {
+          role: 'system',
+          content:
+            'You convert a user voice request for a place into MapFlow ontology tags. Return one JSON object. Use only allowed tags. Do not recommend places.',
+        },
+        {
+          role: 'user',
+          content: JSON.stringify({
+            task:
+              'Select tags explicitly implied by the request. If the request is vague, return an empty tags array.',
+            ontology: placeOntology,
+            transcript: input.transcript,
+          }),
+        },
+      ],
+    }),
+  });
+
+  if (!response.ok) {
+    const errorBody = await response.text();
+    throw new Error(
+      `OpenRouter voice filter failed with ${response.status}: ${errorBody.slice(0, 500)}`,
+    );
+  }
+
+  const payload = (await response.json()) as OpenRouterResponse;
+  const choice = payload.choices?.[0];
+  const content = choice?.message?.content?.trim();
+  if (!content) {
+    throw new Error(
+      `OpenRouter returned an empty voice filter for ${config.openRouterModel}; finish_reason=${choice?.finish_reason ?? 'missing'}.`,
+    );
+  }
+
+  const intent = JSON.parse(content) as PlaceIntent;
+  assertValidPlaceIntent(intent);
+  return intent;
+}
+
+function assertValidPlaceIntent(intent: PlaceIntent) {
+  if (!Array.isArray(intent.tags)) {
+    throw new Error('OpenRouter voice filter tags must be an array.');
+  }
+  for (const tag of intent.tags) {
+    if (!placeOntologyTags.includes(tag)) {
+      throw new Error(`OpenRouter voice filter returned unsupported tag: ${tag}.`);
+    }
+  }
+  if (typeof intent.summary !== 'string') {
+    throw new Error('OpenRouter voice filter summary must be a string.');
+  }
+}

src/audio/deepgram.ts

@@ -0,0 +1,66 @@
+import { config } from '../config.js';
+
+type DeepgramResponse = {
+  results?: {
+    channels?: Array<{
+      alternatives?: Array<{
+        transcript?: string;
+        paragraphs?: {
+          transcript?: string;
+        };
+      }>;
+    }>;
+  };
+};
+
+export async function transcribeAudioContent(input: {
+  audioContentBase64: string;
+  audioMimeType: string;
+}) {
+  if (!config.deepgramApiKey) {
+    throw new Error('DEEPGRAM_API_KEY is required for voice place filters.');
+  }
+  if (input.audioContentBase64.trim() === '') {
+    throw new Error('Voice filter audio is required.');
+  }
+  if (input.audioMimeType.trim() === '') {
+    throw new Error('Voice filter audio MIME type is required.');
+  }
+
+  const params = new URLSearchParams({
+    model: config.deepgramModel,
+    language: config.deepgramLanguage,
+    smart_format: 'true',
+    punctuate: 'true',
+  });
+  const response = await fetch(
+    `https://api.deepgram.com/v1/listen?${params.toString()}`,
+    {
+      method: 'POST',
+      headers: {
+        authorization: `Token ${config.deepgramApiKey}`,
+        'content-type': input.audioMimeType,
+      },
+      body: Buffer.from(input.audioContentBase64, 'base64'),
+    },
+  );
+
+  if (!response.ok) {
+    const body = await response.text();
+    throw new Error(
+      `Deepgram voice filter transcription failed with ${response.status}: ${body.slice(0, 500)}`,
+    );
+  }
+
+  const payload = (await response.json()) as DeepgramResponse;
+  const alternative = payload.results?.channels?.[0]?.alternatives?.[0];
+  const transcript =
+    alternative?.paragraphs?.transcript?.trim() ??
+    alternative?.transcript?.trim() ??
+    '';
+  if (!transcript) {
+    throw new Error('Deepgram returned an empty voice filter transcript.');
+  }
+
+  return transcript;
+}

src/config.ts

@@ -13,6 +13,9 @@ export const config = {
   telegramBotUsername: process.env.TELEGRAM_BOT_USERNAME ?? 'carfteebot',
   telegramWebhookSecret: process.env.TELEGRAM_WEBHOOK_SECRET ?? '',
   googlePlacesApiKey: process.env.GOOGLE_PLACES_API_KEY ?? '',
+  deepgramApiKey: process.env.DEEPGRAM_API_KEY ?? '',
+  deepgramModel: process.env.DEEPGRAM_MODEL ?? 'nova-3',
+  deepgramLanguage: process.env.DEEPGRAM_LANGUAGE ?? 'ru',
   openRouterApiKey: process.env.OPENROUTER_API_KEY ?? '',
   openRouterModel: process.env.OPENROUTER_MODEL ?? 'deepseek/deepseek-v4-flash',
   webAppUrl: process.env.WEB_APP_URL ?? 'https://map.craftee.vn',

src/graphql/places.ts

@@ -1,6 +1,8 @@
 import { prisma } from '../prisma.js';
 import { config } from '../config.js';
 import type { Prisma } from '../generated/prisma/client.js';
+import { classifyPlaceIntent } from '../ai/place-intent.js';
+import { transcribeAudioContent } from '../audio/deepgram.js';
 import {
   fetchGooglePlaceDetails,
   normalizeGooglePlaceDetails,
@@ -25,8 +27,9 @@ type NearbyPlacesInput = {
   radiusMeters: number;
 };
 
-type SearchPlacesInput = {
+type VoicePlaceRecommendationsInput = {
-  query: string;
+  audioContentBase64: string;
+  audioMimeType: string;
 };
 
 type GoogleNearbyPlace = {
@@ -66,20 +69,6 @@ type PlaceWithRecentExperiences = Prisma.PlaceGetPayload<{
   };
 }>;
 
-type AiSearchResponse = {
-  choices?: Array<{
-    finish_reason?: string;
-    message?: {
-      content?: string;
-    };
-  }>;
-};
-
-type AiSearchSelection = {
-  placeIds: string[];
-  message: string;
-};
-
 function serializeVoiceExperience(
   experience: Awaited<ReturnType<typeof prisma.voiceExperience.findMany>>[number],
 ) {
@@ -242,12 +231,12 @@ export async function listVoiceExperiences() {
   return experiences.map(serializeVoiceExperience);
 }
 
-export async function searchPlaces(input: SearchPlacesInput, userId: string) {
+export async function recommendPlacesByVoice(
-  const query = input.query.trim();
+  input: VoicePlaceRecommendationsInput,
-  if (query === '') {
+  userId: string,
-    throw new Error('Place search query is required.');
+) {
-  }
+  const transcript = await transcribeAudioContent(input);
-
+  const intent = await classifyPlaceIntent({ transcript });
   const places = await prisma.place.findMany({
     where: {
       experiences: { some: {} },
@@ -264,42 +253,13 @@ export async function searchPlaces(input: SearchPlacesInput, userId: string) {
     take: 50,
   });
 
-  if (places.length === 0) {
-    return { message: '', places: [] };
-  }
-
-  const selection = await selectPlacesWithAi({
-    query,
-    places: places.map((place) => ({
-      id: place.id,
-      name: place.name,
-      googlePrimaryType: place.googlePrimaryType,
-      googleTypes: place.googleTypes,
-      traits: placeTraits(place),
-      businessStatus: place.googleBusinessStatus,
-      rating: place.googleRating,
-      userRatingCount: place.googleUserRatingCount,
-      currentOpeningHours: place.googleCurrentOpeningHours,
-      isFavorite: place.favoriteUsers.some(
-        (favorite) => favorite.userId === userId,
-      ),
-      reviewSummaries: place.experiences
-        .map((experience) => {
-          const analysis = jsonObject(experience.analysis);
-          const summary = analysis?.summary;
-          return typeof summary === 'string' ? summary : null;
-        })
-        .filter((summary) => summary !== null),
-    })),
-  });
-  const byId = new Map(places.map((place) => [place.id, place]));
-  const selectedPlaces = selection.placeIds
-    .map((id) => byId.get(id))
-    .filter((place) => place !== undefined);
-
   return {
-    message: selection.message,
+    transcript,
-    places: selectedPlaces.map((place) => serializePlace(place, userId)),
+    tags: intent.tags,
+    summary: intent.summary,
+    places: rankPlacesByVoiceTags(places, intent.tags, userId).map((place) =>
+      serializePlace(place, userId),
+    ),
   };
 }
 
@@ -548,95 +508,41 @@ function placeTraits(place: Pick<PlaceWithRecentExperiences, 'experiences'>) {
   return [...tags];
 }
 
+function rankPlacesByVoiceTags(
+  places: PlaceWithRecentExperiences[],
+  tags: string[],
+  userId: string,
+) {
+  if (tags.length === 0) {
+    return [];
+  }
+
+  const requestedTags = new Set(tags);
+  return places
+    .map((place) => {
+      const tagsForPlace = placeTraits(place);
+      const matchedTags = tagsForPlace.filter((tag) => requestedTags.has(tag));
+      const favoriteBoost = place.favoriteUsers.some(
+        (favorite) => favorite.userId === userId,
+      )
+        ? 0.25
+        : 0;
+      const ratingBoost = place.googleRating ? place.googleRating / 10 : 0;
+      return {
+        place,
+        matchedCount: matchedTags.length,
+        score: matchedTags.length * 10 + favoriteBoost + ratingBoost,
+      };
+    })
+    .filter((item) => item.matchedCount > 0)
+    .sort((left, right) => right.score - left.score)
+    .slice(0, 12)
+    .map((item) => item.place);
+}
+
 function jsonObject(value: unknown): Record<string, unknown> | null {
   if (!value || Array.isArray(value) || typeof value !== 'object') {
     return null;
   }
   return value as Record<string, unknown>;
 }
-
-async function selectPlacesWithAi(input: {
-  query: string;
-  places: Array<Record<string, unknown>>;
-}): Promise<AiSearchSelection> {
-  if (!config.openRouterApiKey) {
-    throw new Error('OPENROUTER_API_KEY is required for AI place search.');
-  }
-
-  const response = await fetch('https://openrouter.ai/api/v1/chat/completions', {
-    method: 'POST',
-    headers: {
-      authorization: `Bearer ${config.openRouterApiKey}`,
-      'content-type': 'application/json',
-      'http-referer': config.webAppUrl,
-      'x-title': 'MapFlow',
-    },
-    body: JSON.stringify({
-      model: config.openRouterModel,
-      provider: { require_parameters: true },
-      temperature: 0.1,
-      max_tokens: 1200,
-      response_format: {
-        type: 'json_schema',
-        json_schema: {
-          name: 'place_search',
-          strict: true,
-          schema: {
-            type: 'object',
-            additionalProperties: false,
-            required: ['placeIds', 'message'],
-            properties: {
-              placeIds: {
-                type: 'array',
-                maxItems: 12,
-                items: { type: 'string' },
-              },
-              message: { type: 'string' },
-            },
-          },
-        },
-      },
-      messages: [
-        {
-          role: 'system',
-          content:
-            'You rank known places for a map UI. Use only provided place ids. Return one valid JSON object. Do not invent places.',
-        },
-        {
-          role: 'user',
-          content: JSON.stringify({
-            task:
-              'Select places that best match the user search. Prefer favorites and open places only when relevant to the query.',
-            query: input.query,
-            places: input.places,
-          }),
-        },
-      ],
-    }),
-  });
-
-  if (!response.ok) {
-    const errorBody = await response.text();
-    throw new Error(
-      `OpenRouter place search failed with ${response.status}: ${errorBody.slice(0, 500)}`,
-    );
-  }
-
-  const payload = (await response.json()) as AiSearchResponse;
-  const choice = payload.choices?.[0];
-  const content = choice?.message?.content?.trim();
-  if (!content) {
-    throw new Error(
-      `OpenRouter returned an empty place search for ${config.openRouterModel}; finish_reason=${choice?.finish_reason ?? 'missing'}.`,
-    );
-  }
-
-  const selection = JSON.parse(content) as AiSearchSelection;
-  if (!Array.isArray(selection.placeIds)) {
-    throw new Error('OpenRouter place search placeIds must be an array.');
-  }
-  if (typeof selection.message !== 'string') {
-    throw new Error('OpenRouter place search message must be a string.');
-  }
-  return selection;
-}

src/graphql/schema.ts

@@ -17,8 +17,8 @@ import {
   listNearbyPlaces,
   listPlaces,
   listVoiceExperiences,
+  recommendPlacesByVoice,
   removeFavoritePlace,
-  searchPlaces,
 } from './places.js';
 import { createVoiceExperience } from './voice-experiences.js';
 
@@ -57,15 +57,6 @@ export const resolvers = {
       const user = await requireTelegramUser(graphqlContext);
       return listNearbyPlaces(args.input, user.id);
     },
-    searchPlaces: async (
-      _: unknown,
-      args: { input: Parameters<typeof searchPlaces>[0] },
-      context: unknown,
-    ) => {
-      const graphqlContext = context as GraphqlContext;
-      const user = await requireTelegramUser(graphqlContext);
-      return searchPlaces(args.input, user.id);
-    },
     voiceExperiences: async (_: unknown, __: unknown, context: unknown) => {
       const graphqlContext = context as GraphqlContext;
       await requireAdminTelegramUser(graphqlContext);
@@ -93,6 +84,15 @@ export const resolvers = {
       const user = await requireTelegramUser(graphqlContext);
       return createVoiceExperience(args.input, user.id);
     },
+    recommendPlacesByVoice: async (
+      _: unknown,
+      args: { input: Parameters<typeof recommendPlacesByVoice>[0] },
+      context: unknown,
+    ) => {
+      const graphqlContext = context as GraphqlContext;
+      const user = await requireTelegramUser(graphqlContext);
+      return recommendPlacesByVoice(args.input, user.id);
+    },
     addFavoritePlace: async (
       _: unknown,
       args: { placeId: string },

src/graphql/type-defs.ts

@@ -79,8 +79,9 @@ export const typeDefs = /* GraphQL */ `
     radiusMeters: Int!
   }
 
-  input SearchPlacesInput {
+  input VoicePlaceRecommendationsInput {
-    query: String!
+    audioContentBase64: String!
+    audioMimeType: String!
   }
 
   input AuthenticateTelegramInput {
@@ -112,8 +113,10 @@ export const typeDefs = /* GraphQL */ `
     user: User!
   }
 
-  type SearchPlacesPayload {
+  type VoicePlaceRecommendationsPayload {
-    message: String!
+    transcript: String!
+    tags: [String!]!
+    summary: String!
     places: [Place!]!
   }
 
@@ -123,7 +126,6 @@ export const typeDefs = /* GraphQL */ `
     places: [Place!]!
     favoritePlaces: [Place!]!
     nearbyPlaces(input: NearbyPlacesInput!): [Place!]!
-    searchPlaces(input: SearchPlacesInput!): SearchPlacesPayload!
     voiceExperiences: [VoiceExperience!]!
   }
 
@@ -133,6 +135,7 @@ export const typeDefs = /* GraphQL */ `
     authenticateTelegram(input: AuthenticateTelegramInput!): AuthPayload!
     authenticateTelegramLogin(input: AuthenticateTelegramLoginInput!): AuthPayload!
     createVoiceExperience(input: CreateVoiceExperienceInput!): VoiceExperience!
+    recommendPlacesByVoice(input: VoicePlaceRecommendationsInput!): VoicePlaceRecommendationsPayload!
     addFavoritePlace(placeId: ID!): Place!
     removeFavoritePlace(placeId: ID!): Place!
   }

src/ontology/place-ontology.ts

@@ -0,0 +1,103 @@
+export type OntologyLeaf = {
+  id: string;
+  label: string;
+  keywords: string[];
+};
+
+export type OntologyAxis = {
+  id: string;
+  label: string;
+  leaves: OntologyLeaf[];
+};
+
+export const placeOntology: OntologyAxis[] = [
+  {
+    id: 'energy',
+    label: 'энергия',
+    leaves: [
+      {
+        id: 'calm',
+        label: 'спокойное',
+        keywords: ['тихо', 'спокойно', 'медленно', 'мягко', 'без шума'],
+      },
+      {
+        id: 'dynamic',
+        label: 'живое',
+        keywords: ['движ', 'громко', 'оживленно', 'толпа', 'активно'],
+      },
+    ],
+  },
+  {
+    id: 'privacy',
+    label: 'приватность',
+    leaves: [
+      {
+        id: 'intimate',
+        label: 'камерное',
+        keywords: ['уютно', 'камерно', 'маленькое', 'спрятаться', 'приватно'],
+      },
+      {
+        id: 'open',
+        label: 'открытое',
+        keywords: ['просторно', 'открыто', 'видно', 'публично', 'воздух'],
+      },
+    ],
+  },
+  {
+    id: 'sociality',
+    label: 'социальность',
+    leaves: [
+      {
+        id: 'solo',
+        label: 'для себя',
+        keywords: ['одному', 'поработать', 'почитать', 'фокус', 'ноутбук'],
+      },
+      {
+        id: 'group',
+        label: 'для компании',
+        keywords: ['друзья', 'компания', 'свидание', 'поговорить', 'вечер'],
+      },
+    ],
+  },
+  {
+    id: 'function',
+    label: 'сценарий',
+    leaves: [
+      {
+        id: 'reset',
+        label: 'выдохнуть',
+        keywords: ['отдохнуть', 'выдохнуть', 'перезагрузиться', 'паузу'],
+      },
+      {
+        id: 'impress',
+        label: 'впечатлить',
+        keywords: ['красиво', 'особенно', 'вау', 'впечатлить', 'необычно'],
+      },
+      {
+        id: 'transit',
+        label: 'транзитное',
+        keywords: ['быстро', 'рядом', 'по пути', 'забежать', 'перекусить'],
+      },
+    ],
+  },
+  {
+    id: 'aesthetic',
+    label: 'образ',
+    leaves: [
+      {
+        id: 'clean',
+        label: 'чистое',
+        keywords: ['чисто', 'минимально', 'светло', 'просто', 'аккуратно'],
+      },
+      {
+        id: 'expressive',
+        label: 'выразительное',
+        keywords: ['дизайн', 'арт', 'фактура', 'ярко', 'атмосферно'],
+      },
+    ],
+  },
+];
+
+export const placeOntologyTags = placeOntology.flatMap((axis) =>
+  axis.leaves.map((leaf) => `${axis.id}:${leaf.id}`),
+);