diff --git a/.env.example b/.env.example index e21882e..8206996 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,8 @@ DATABASE_URL=postgresql://mapflow:mapflow@localhost:5432/mapflow HATCHET_CLIENT_TOKEN= +DEEPGRAM_API_KEY= +DEEPGRAM_MODEL=nova-3 +DEEPGRAM_LANGUAGE=ru OPENROUTER_API_KEY= OPENROUTER_MODEL=deepseek/deepseek-v4-flash HOST=0.0.0.0 diff --git a/graphql-contracts b/graphql-contracts index 39f32e3..d694c7d 160000 --- a/graphql-contracts +++ b/graphql-contracts @@ -1 +1 @@ -Subproject commit 39f32e3d3663e3ea724573d54973984645ae1853 +Subproject commit d694c7d78c3e234ae19bf41af6b0e555d048109c diff --git a/src/ai/place-intent.ts b/src/ai/place-intent.ts new file mode 100644 index 0000000..d056d58 --- /dev/null +++ b/src/ai/place-intent.ts @@ -0,0 +1,110 @@ +import { config } from '../config.js'; +import { placeOntology, placeOntologyTags } from '../ontology/place-ontology.js'; + +type OpenRouterResponse = { + choices?: Array<{ + finish_reason?: string; + message?: { + content?: string; + }; + }>; +}; + +export type PlaceIntent = { + tags: string[]; + summary: string; +}; + +export async function classifyPlaceIntent(input: { + transcript: string; +}): Promise { + if (!config.openRouterApiKey) { + throw new Error('OPENROUTER_API_KEY is required for voice place filters.'); + } + + const response = await fetch('https://openrouter.ai/api/v1/chat/completions', { + method: 'POST', + headers: { + authorization: `Bearer ${config.openRouterApiKey}`, + 'content-type': 'application/json', + 'http-referer': config.webAppUrl, + 'x-title': 'MapFlow', + }, + body: JSON.stringify({ + model: config.openRouterModel, + provider: { require_parameters: true }, + temperature: 0.1, + max_tokens: 900, + response_format: { + type: 'json_schema', + json_schema: { + name: 'place_voice_filter', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + required: ['tags', 'summary'], + properties: { + tags: { + type: 'array', + maxItems: 8, + items: { type: 'string', enum: placeOntologyTags }, + }, + summary: { type: 'string' }, + }, + }, + }, + }, + messages: [ + { + role: 'system', + content: + 'You convert a user voice request for a place into MapFlow ontology tags. Return one JSON object. Use only allowed tags. Do not recommend places.', + }, + { + role: 'user', + content: JSON.stringify({ + task: + 'Select tags explicitly implied by the request. If the request is vague, return an empty tags array.', + ontology: placeOntology, + transcript: input.transcript, + }), + }, + ], + }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + throw new Error( + `OpenRouter voice filter failed with ${response.status}: ${errorBody.slice(0, 500)}`, + ); + } + + const payload = (await response.json()) as OpenRouterResponse; + const choice = payload.choices?.[0]; + const content = choice?.message?.content?.trim(); + if (!content) { + throw new Error( + `OpenRouter returned an empty voice filter for ${config.openRouterModel}; finish_reason=${choice?.finish_reason ?? 'missing'}.`, + ); + } + + const intent = JSON.parse(content) as PlaceIntent; + assertValidPlaceIntent(intent); + return intent; +} + +function assertValidPlaceIntent(intent: PlaceIntent) { + if (!Array.isArray(intent.tags)) { + throw new Error('OpenRouter voice filter tags must be an array.'); + } + for (const tag of intent.tags) { + if (!placeOntologyTags.includes(tag)) { + throw new Error(`OpenRouter voice filter returned unsupported tag: ${tag}.`); + } + } + if (typeof intent.summary !== 'string') { + throw new Error('OpenRouter voice filter summary must be a string.'); + } +} diff --git a/src/audio/deepgram.ts b/src/audio/deepgram.ts new file mode 100644 index 0000000..973dbc6 --- /dev/null +++ b/src/audio/deepgram.ts @@ -0,0 +1,66 @@ +import { config } from '../config.js'; + +type DeepgramResponse = { + results?: { + channels?: Array<{ + alternatives?: Array<{ + transcript?: string; + paragraphs?: { + transcript?: string; + }; + }>; + }>; + }; +}; + +export async function transcribeAudioContent(input: { + audioContentBase64: string; + audioMimeType: string; +}) { + if (!config.deepgramApiKey) { + throw new Error('DEEPGRAM_API_KEY is required for voice place filters.'); + } + if (input.audioContentBase64.trim() === '') { + throw new Error('Voice filter audio is required.'); + } + if (input.audioMimeType.trim() === '') { + throw new Error('Voice filter audio MIME type is required.'); + } + + const params = new URLSearchParams({ + model: config.deepgramModel, + language: config.deepgramLanguage, + smart_format: 'true', + punctuate: 'true', + }); + const response = await fetch( + `https://api.deepgram.com/v1/listen?${params.toString()}`, + { + method: 'POST', + headers: { + authorization: `Token ${config.deepgramApiKey}`, + 'content-type': input.audioMimeType, + }, + body: Buffer.from(input.audioContentBase64, 'base64'), + }, + ); + + if (!response.ok) { + const body = await response.text(); + throw new Error( + `Deepgram voice filter transcription failed with ${response.status}: ${body.slice(0, 500)}`, + ); + } + + const payload = (await response.json()) as DeepgramResponse; + const alternative = payload.results?.channels?.[0]?.alternatives?.[0]; + const transcript = + alternative?.paragraphs?.transcript?.trim() ?? + alternative?.transcript?.trim() ?? + ''; + if (!transcript) { + throw new Error('Deepgram returned an empty voice filter transcript.'); + } + + return transcript; +} diff --git a/src/config.ts b/src/config.ts index 0c33e01..4f0bfe0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -13,6 +13,9 @@ export const config = { telegramBotUsername: process.env.TELEGRAM_BOT_USERNAME ?? 'carfteebot', telegramWebhookSecret: process.env.TELEGRAM_WEBHOOK_SECRET ?? '', googlePlacesApiKey: process.env.GOOGLE_PLACES_API_KEY ?? '', + deepgramApiKey: process.env.DEEPGRAM_API_KEY ?? '', + deepgramModel: process.env.DEEPGRAM_MODEL ?? 'nova-3', + deepgramLanguage: process.env.DEEPGRAM_LANGUAGE ?? 'ru', openRouterApiKey: process.env.OPENROUTER_API_KEY ?? '', openRouterModel: process.env.OPENROUTER_MODEL ?? 'deepseek/deepseek-v4-flash', webAppUrl: process.env.WEB_APP_URL ?? 'https://map.craftee.vn', diff --git a/src/graphql/places.ts b/src/graphql/places.ts index 6984f6c..d9607c2 100644 --- a/src/graphql/places.ts +++ b/src/graphql/places.ts @@ -1,6 +1,8 @@ import { prisma } from '../prisma.js'; import { config } from '../config.js'; import type { Prisma } from '../generated/prisma/client.js'; +import { classifyPlaceIntent } from '../ai/place-intent.js'; +import { transcribeAudioContent } from '../audio/deepgram.js'; import { fetchGooglePlaceDetails, normalizeGooglePlaceDetails, @@ -25,8 +27,9 @@ type NearbyPlacesInput = { radiusMeters: number; }; -type SearchPlacesInput = { - query: string; +type VoicePlaceRecommendationsInput = { + audioContentBase64: string; + audioMimeType: string; }; type GoogleNearbyPlace = { @@ -66,20 +69,6 @@ type PlaceWithRecentExperiences = Prisma.PlaceGetPayload<{ }; }>; -type AiSearchResponse = { - choices?: Array<{ - finish_reason?: string; - message?: { - content?: string; - }; - }>; -}; - -type AiSearchSelection = { - placeIds: string[]; - message: string; -}; - function serializeVoiceExperience( experience: Awaited>[number], ) { @@ -242,12 +231,12 @@ export async function listVoiceExperiences() { return experiences.map(serializeVoiceExperience); } -export async function searchPlaces(input: SearchPlacesInput, userId: string) { - const query = input.query.trim(); - if (query === '') { - throw new Error('Place search query is required.'); - } - +export async function recommendPlacesByVoice( + input: VoicePlaceRecommendationsInput, + userId: string, +) { + const transcript = await transcribeAudioContent(input); + const intent = await classifyPlaceIntent({ transcript }); const places = await prisma.place.findMany({ where: { experiences: { some: {} }, @@ -264,42 +253,13 @@ export async function searchPlaces(input: SearchPlacesInput, userId: string) { take: 50, }); - if (places.length === 0) { - return { message: '', places: [] }; - } - - const selection = await selectPlacesWithAi({ - query, - places: places.map((place) => ({ - id: place.id, - name: place.name, - googlePrimaryType: place.googlePrimaryType, - googleTypes: place.googleTypes, - traits: placeTraits(place), - businessStatus: place.googleBusinessStatus, - rating: place.googleRating, - userRatingCount: place.googleUserRatingCount, - currentOpeningHours: place.googleCurrentOpeningHours, - isFavorite: place.favoriteUsers.some( - (favorite) => favorite.userId === userId, - ), - reviewSummaries: place.experiences - .map((experience) => { - const analysis = jsonObject(experience.analysis); - const summary = analysis?.summary; - return typeof summary === 'string' ? summary : null; - }) - .filter((summary) => summary !== null), - })), - }); - const byId = new Map(places.map((place) => [place.id, place])); - const selectedPlaces = selection.placeIds - .map((id) => byId.get(id)) - .filter((place) => place !== undefined); - return { - message: selection.message, - places: selectedPlaces.map((place) => serializePlace(place, userId)), + transcript, + tags: intent.tags, + summary: intent.summary, + places: rankPlacesByVoiceTags(places, intent.tags, userId).map((place) => + serializePlace(place, userId), + ), }; } @@ -548,95 +508,41 @@ function placeTraits(place: Pick) { return [...tags]; } +function rankPlacesByVoiceTags( + places: PlaceWithRecentExperiences[], + tags: string[], + userId: string, +) { + if (tags.length === 0) { + return []; + } + + const requestedTags = new Set(tags); + return places + .map((place) => { + const tagsForPlace = placeTraits(place); + const matchedTags = tagsForPlace.filter((tag) => requestedTags.has(tag)); + const favoriteBoost = place.favoriteUsers.some( + (favorite) => favorite.userId === userId, + ) + ? 0.25 + : 0; + const ratingBoost = place.googleRating ? place.googleRating / 10 : 0; + return { + place, + matchedCount: matchedTags.length, + score: matchedTags.length * 10 + favoriteBoost + ratingBoost, + }; + }) + .filter((item) => item.matchedCount > 0) + .sort((left, right) => right.score - left.score) + .slice(0, 12) + .map((item) => item.place); +} + function jsonObject(value: unknown): Record | null { if (!value || Array.isArray(value) || typeof value !== 'object') { return null; } return value as Record; } - -async function selectPlacesWithAi(input: { - query: string; - places: Array>; -}): Promise { - if (!config.openRouterApiKey) { - throw new Error('OPENROUTER_API_KEY is required for AI place search.'); - } - - const response = await fetch('https://openrouter.ai/api/v1/chat/completions', { - method: 'POST', - headers: { - authorization: `Bearer ${config.openRouterApiKey}`, - 'content-type': 'application/json', - 'http-referer': config.webAppUrl, - 'x-title': 'MapFlow', - }, - body: JSON.stringify({ - model: config.openRouterModel, - provider: { require_parameters: true }, - temperature: 0.1, - max_tokens: 1200, - response_format: { - type: 'json_schema', - json_schema: { - name: 'place_search', - strict: true, - schema: { - type: 'object', - additionalProperties: false, - required: ['placeIds', 'message'], - properties: { - placeIds: { - type: 'array', - maxItems: 12, - items: { type: 'string' }, - }, - message: { type: 'string' }, - }, - }, - }, - }, - messages: [ - { - role: 'system', - content: - 'You rank known places for a map UI. Use only provided place ids. Return one valid JSON object. Do not invent places.', - }, - { - role: 'user', - content: JSON.stringify({ - task: - 'Select places that best match the user search. Prefer favorites and open places only when relevant to the query.', - query: input.query, - places: input.places, - }), - }, - ], - }), - }); - - if (!response.ok) { - const errorBody = await response.text(); - throw new Error( - `OpenRouter place search failed with ${response.status}: ${errorBody.slice(0, 500)}`, - ); - } - - const payload = (await response.json()) as AiSearchResponse; - const choice = payload.choices?.[0]; - const content = choice?.message?.content?.trim(); - if (!content) { - throw new Error( - `OpenRouter returned an empty place search for ${config.openRouterModel}; finish_reason=${choice?.finish_reason ?? 'missing'}.`, - ); - } - - const selection = JSON.parse(content) as AiSearchSelection; - if (!Array.isArray(selection.placeIds)) { - throw new Error('OpenRouter place search placeIds must be an array.'); - } - if (typeof selection.message !== 'string') { - throw new Error('OpenRouter place search message must be a string.'); - } - return selection; -} diff --git a/src/graphql/schema.ts b/src/graphql/schema.ts index e7ca247..81e40a1 100644 --- a/src/graphql/schema.ts +++ b/src/graphql/schema.ts @@ -17,8 +17,8 @@ import { listNearbyPlaces, listPlaces, listVoiceExperiences, + recommendPlacesByVoice, removeFavoritePlace, - searchPlaces, } from './places.js'; import { createVoiceExperience } from './voice-experiences.js'; @@ -57,15 +57,6 @@ export const resolvers = { const user = await requireTelegramUser(graphqlContext); return listNearbyPlaces(args.input, user.id); }, - searchPlaces: async ( - _: unknown, - args: { input: Parameters[0] }, - context: unknown, - ) => { - const graphqlContext = context as GraphqlContext; - const user = await requireTelegramUser(graphqlContext); - return searchPlaces(args.input, user.id); - }, voiceExperiences: async (_: unknown, __: unknown, context: unknown) => { const graphqlContext = context as GraphqlContext; await requireAdminTelegramUser(graphqlContext); @@ -93,6 +84,15 @@ export const resolvers = { const user = await requireTelegramUser(graphqlContext); return createVoiceExperience(args.input, user.id); }, + recommendPlacesByVoice: async ( + _: unknown, + args: { input: Parameters[0] }, + context: unknown, + ) => { + const graphqlContext = context as GraphqlContext; + const user = await requireTelegramUser(graphqlContext); + return recommendPlacesByVoice(args.input, user.id); + }, addFavoritePlace: async ( _: unknown, args: { placeId: string }, diff --git a/src/graphql/type-defs.ts b/src/graphql/type-defs.ts index b897934..67bc937 100644 --- a/src/graphql/type-defs.ts +++ b/src/graphql/type-defs.ts @@ -79,8 +79,9 @@ export const typeDefs = /* GraphQL */ ` radiusMeters: Int! } - input SearchPlacesInput { - query: String! + input VoicePlaceRecommendationsInput { + audioContentBase64: String! + audioMimeType: String! } input AuthenticateTelegramInput { @@ -112,8 +113,10 @@ export const typeDefs = /* GraphQL */ ` user: User! } - type SearchPlacesPayload { - message: String! + type VoicePlaceRecommendationsPayload { + transcript: String! + tags: [String!]! + summary: String! places: [Place!]! } @@ -123,7 +126,6 @@ export const typeDefs = /* GraphQL */ ` places: [Place!]! favoritePlaces: [Place!]! nearbyPlaces(input: NearbyPlacesInput!): [Place!]! - searchPlaces(input: SearchPlacesInput!): SearchPlacesPayload! voiceExperiences: [VoiceExperience!]! } @@ -133,6 +135,7 @@ export const typeDefs = /* GraphQL */ ` authenticateTelegram(input: AuthenticateTelegramInput!): AuthPayload! authenticateTelegramLogin(input: AuthenticateTelegramLoginInput!): AuthPayload! createVoiceExperience(input: CreateVoiceExperienceInput!): VoiceExperience! + recommendPlacesByVoice(input: VoicePlaceRecommendationsInput!): VoicePlaceRecommendationsPayload! addFavoritePlace(placeId: ID!): Place! removeFavoritePlace(placeId: ID!): Place! } diff --git a/src/ontology/place-ontology.ts b/src/ontology/place-ontology.ts new file mode 100644 index 0000000..af06d3f --- /dev/null +++ b/src/ontology/place-ontology.ts @@ -0,0 +1,103 @@ +export type OntologyLeaf = { + id: string; + label: string; + keywords: string[]; +}; + +export type OntologyAxis = { + id: string; + label: string; + leaves: OntologyLeaf[]; +}; + +export const placeOntology: OntologyAxis[] = [ + { + id: 'energy', + label: 'энергия', + leaves: [ + { + id: 'calm', + label: 'спокойное', + keywords: ['тихо', 'спокойно', 'медленно', 'мягко', 'без шума'], + }, + { + id: 'dynamic', + label: 'живое', + keywords: ['движ', 'громко', 'оживленно', 'толпа', 'активно'], + }, + ], + }, + { + id: 'privacy', + label: 'приватность', + leaves: [ + { + id: 'intimate', + label: 'камерное', + keywords: ['уютно', 'камерно', 'маленькое', 'спрятаться', 'приватно'], + }, + { + id: 'open', + label: 'открытое', + keywords: ['просторно', 'открыто', 'видно', 'публично', 'воздух'], + }, + ], + }, + { + id: 'sociality', + label: 'социальность', + leaves: [ + { + id: 'solo', + label: 'для себя', + keywords: ['одному', 'поработать', 'почитать', 'фокус', 'ноутбук'], + }, + { + id: 'group', + label: 'для компании', + keywords: ['друзья', 'компания', 'свидание', 'поговорить', 'вечер'], + }, + ], + }, + { + id: 'function', + label: 'сценарий', + leaves: [ + { + id: 'reset', + label: 'выдохнуть', + keywords: ['отдохнуть', 'выдохнуть', 'перезагрузиться', 'паузу'], + }, + { + id: 'impress', + label: 'впечатлить', + keywords: ['красиво', 'особенно', 'вау', 'впечатлить', 'необычно'], + }, + { + id: 'transit', + label: 'транзитное', + keywords: ['быстро', 'рядом', 'по пути', 'забежать', 'перекусить'], + }, + ], + }, + { + id: 'aesthetic', + label: 'образ', + leaves: [ + { + id: 'clean', + label: 'чистое', + keywords: ['чисто', 'минимально', 'светло', 'просто', 'аккуратно'], + }, + { + id: 'expressive', + label: 'выразительное', + keywords: ['дизайн', 'арт', 'фактура', 'ярко', 'атмосферно'], + }, + ], + }, +]; + +export const placeOntologyTags = placeOntology.flatMap((axis) => + axis.leaves.map((leaf) => `${axis.id}:${leaf.id}`), +);