Restructure omni services and add Chatwoot research snapshot

research/chatwoot/enterprise/app/jobs/captain/conversation/response_builder_job.rb

@@ -0,0 +1,147 @@
+class Captain::Conversation::ResponseBuilderJob < ApplicationJob
+  MAX_MESSAGE_LENGTH = 10_000
+  retry_on ActiveStorage::FileNotFoundError, attempts: 3, wait: 2.seconds
+  retry_on Faraday::BadRequestError, attempts: 3, wait: 2.seconds
+
+  def perform(conversation, assistant)
+    @conversation = conversation
+    @inbox = conversation.inbox
+    @assistant = assistant
+
+    Current.executed_by = @assistant
+
+    if captain_v2_enabled?
+      generate_response_with_v2
+    else
+      generate_and_process_response
+    end
+  rescue StandardError => e
+    raise e if e.is_a?(ActiveStorage::FileNotFoundError) || e.is_a?(Faraday::BadRequestError)
+
+    handle_error(e)
+  ensure
+    Current.executed_by = nil
+  end
+
+  private
+
+  delegate :account, :inbox, to: :@conversation
+
+  def generate_and_process_response
+    @response = Captain::Llm::AssistantChatService.new(assistant: @assistant, conversation_id: @conversation.display_id).generate_response(
+      message_history: collect_previous_messages
+    )
+    process_response
+  end
+
+  def generate_response_with_v2
+    @response = Captain::Assistant::AgentRunnerService.new(assistant: @assistant, conversation: @conversation).generate_response(
+      message_history: collect_previous_messages
+    )
+    process_response
+  end
+
+  def process_response
+    ActiveRecord::Base.transaction do
+      if handoff_requested?
+        process_action('handoff')
+      else
+        create_messages
+        Rails.logger.info("[CAPTAIN][ResponseBuilderJob] Incrementing response usage for #{account.id}")
+        account.increment_response_usage
+      end
+    end
+  end
+
+  def collect_previous_messages
+    @conversation
+      .messages
+      .where(message_type: [:incoming, :outgoing])
+      .where(private: false)
+      .map do |message|
+      message_hash = {
+        content: prepare_multimodal_message_content(message),
+        role: determine_role(message)
+      }
+
+      # Include agent_name if present in additional_attributes
+      message_hash[:agent_name] = message.additional_attributes['agent_name'] if message.additional_attributes&.dig('agent_name').present?
+
+      message_hash
+    end
+  end
+
+  def determine_role(message)
+    message.message_type == 'incoming' ? 'user' : 'assistant'
+  end
+
+  def prepare_multimodal_message_content(message)
+    Captain::OpenAiMessageBuilderService.new(message: message).generate_content
+  end
+
+  def handoff_requested?
+    @response['response'] == 'conversation_handoff'
+  end
+
+  def process_action(action)
+    case action
+    when 'handoff'
+      I18n.with_locale(@assistant.account.locale) do
+        create_handoff_message
+        @conversation.bot_handoff!
+        send_out_of_office_message_if_applicable
+      end
+    end
+  end
+
+  def send_out_of_office_message_if_applicable
+    # Campaign conversations should never receive OOO templates — the campaign itself
+    # serves as the initial outreach, and OOO would be confusing in that context.
+    return if @conversation.campaign.present?
+
+    ::MessageTemplates::Template::OutOfOffice.perform_if_applicable(@conversation)
+  end
+
+  def create_handoff_message
+    create_outgoing_message(
+      @assistant.config['handoff_message'].presence || I18n.t('conversations.captain.handoff')
+    )
+  end
+
+  def create_messages
+    validate_message_content!(@response['response'])
+    create_outgoing_message(@response['response'], agent_name: @response['agent_name'])
+  end
+
+  def validate_message_content!(content)
+    raise ArgumentError, 'Message content cannot be blank' if content.blank?
+  end
+
+  def create_outgoing_message(message_content, agent_name: nil)
+    additional_attrs = {}
+    additional_attrs[:agent_name] = agent_name if agent_name.present?
+
+    @conversation.messages.create!(
+      message_type: :outgoing,
+      account_id: account.id,
+      inbox_id: inbox.id,
+      sender: @assistant,
+      content: message_content,
+      additional_attributes: additional_attrs
+    )
+  end
+
+  def handle_error(error)
+    log_error(error)
+    process_action('handoff')
+    true
+  end
+
+  def log_error(error)
+    ChatwootExceptionTracker.new(error, account: account).capture_exception
+  end
+
+  def captain_v2_enabled?
+    account.feature_enabled?('captain_integration_v2')
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/copilot/response_job.rb

@@ -0,0 +1,28 @@
+class Captain::Copilot::ResponseJob < ApplicationJob
+  queue_as :default
+
+  def perform(assistant:, conversation_id:, user_id:, copilot_thread_id:, message:)
+    Rails.logger.info("#{self.class.name} Copilot response job for assistant_id=#{assistant.id} user_id=#{user_id}")
+    generate_chat_response(
+      assistant: assistant,
+      conversation_id: conversation_id,
+      user_id: user_id,
+      copilot_thread_id: copilot_thread_id,
+      message: message
+    )
+  end
+
+  private
+
+  def generate_chat_response(assistant:, conversation_id:, user_id:, copilot_thread_id:, message:)
+    service = Captain::Copilot::ChatService.new(
+      assistant,
+      user_id: user_id,
+      copilot_thread_id: copilot_thread_id,
+      conversation_id: conversation_id
+    )
+    # When using copilot_thread, message is already in previous_history
+    # Pass nil to avoid duplicate
+    service.generate_response(copilot_thread_id.present? ? nil : message)
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/documents/crawl_job.rb

@@ -0,0 +1,61 @@
+class Captain::Documents::CrawlJob < ApplicationJob
+  queue_as :low
+
+  def perform(document)
+    if document.pdf_document?
+      perform_pdf_processing(document)
+    elsif InstallationConfig.find_by(name: 'CAPTAIN_FIRECRAWL_API_KEY')&.value.present?
+      perform_firecrawl_crawl(document)
+    else
+      perform_simple_crawl(document)
+    end
+  end
+
+  private
+
+  include Captain::FirecrawlHelper
+
+  def perform_pdf_processing(document)
+    Captain::Llm::PdfProcessingService.new(document).process
+    document.update!(status: :available)
+  rescue StandardError => e
+    Rails.logger.error I18n.t('captain.documents.pdf_processing_failed', document_id: document.id, error: e.message)
+    raise # Re-raise to let job framework handle retry logic
+  end
+
+  def perform_simple_crawl(document)
+    page_links = Captain::Tools::SimplePageCrawlService.new(document.external_link).page_links
+
+    page_links.each do |page_link|
+      Captain::Tools::SimplePageCrawlParserJob.perform_later(
+        assistant_id: document.assistant_id,
+        page_link: page_link
+      )
+    end
+
+    Captain::Tools::SimplePageCrawlParserJob.perform_later(
+      assistant_id: document.assistant_id,
+      page_link: document.external_link
+    )
+  end
+
+  def perform_firecrawl_crawl(document)
+    captain_usage_limits = document.account.usage_limits[:captain] || {}
+    document_limit = captain_usage_limits[:documents] || {}
+    crawl_limit = [document_limit[:current_available] || 10, 500].min
+
+    Captain::Tools::FirecrawlService
+      .new
+      .perform(
+        document.external_link,
+        firecrawl_webhook_url(document),
+        crawl_limit
+      )
+  end
+
+  def firecrawl_webhook_url(document)
+    webhook_url = Rails.application.routes.url_helpers.enterprise_webhooks_firecrawl_url
+
+    "#{webhook_url}?assistant_id=#{document.assistant_id}&token=#{generate_firecrawl_token(document.assistant_id, document.account_id)}"
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/documents/response_builder_job.rb

@@ -0,0 +1,78 @@
+class Captain::Documents::ResponseBuilderJob < ApplicationJob
+  queue_as :low
+
+  def perform(document, options = {})
+    reset_previous_responses(document)
+
+    faqs = generate_faqs(document, options)
+    create_responses_from_faqs(faqs, document)
+  end
+
+  private
+
+  def generate_faqs(document, options)
+    if should_use_pagination?(document)
+      generate_paginated_faqs(document, options)
+    else
+      generate_standard_faqs(document)
+    end
+  end
+
+  def generate_paginated_faqs(document, options)
+    service = build_paginated_service(document, options)
+    faqs = service.generate
+    store_paginated_metadata(document, service)
+    faqs
+  end
+
+  def generate_standard_faqs(document)
+    Captain::Llm::FaqGeneratorService.new(document.content, document.account.locale_english_name, account_id: document.account_id).generate
+  end
+
+  def build_paginated_service(document, options)
+    Captain::Llm::PaginatedFaqGeneratorService.new(
+      document,
+      pages_per_chunk: options[:pages_per_chunk],
+      max_pages: options[:max_pages],
+      language: document.account.locale_english_name
+    )
+  end
+
+  def store_paginated_metadata(document, service)
+    document.update!(
+      metadata: (document.metadata || {}).merge(
+        'faq_generation' => {
+          'method' => 'paginated',
+          'pages_processed' => service.total_pages_processed,
+          'iterations' => service.iterations_completed,
+          'timestamp' => Time.current.iso8601
+        }
+      )
+    )
+  end
+
+  def create_responses_from_faqs(faqs, document)
+    faqs.each { |faq| create_response(faq, document) }
+  end
+
+  def should_use_pagination?(document)
+    # Auto-detect when to use pagination
+    # For now, use pagination for PDFs with OpenAI file ID
+    document.pdf_document? && document.openai_file_id.present?
+  end
+
+  def reset_previous_responses(response_document)
+    response_document.responses.destroy_all
+  end
+
+  def create_response(faq, document)
+    document.responses.create!(
+      question: faq['question'],
+      answer: faq['answer'],
+      assistant: document.assistant,
+      documentable: document
+    )
+  rescue ActiveRecord::RecordInvalid => e
+    Rails.logger.error I18n.t('captain.documents.response_creation_error', error: e.message)
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/inbox_pending_conversations_resolution_job.rb

@@ -0,0 +1,32 @@
+class Captain::InboxPendingConversationsResolutionJob < ApplicationJob
+  queue_as :low
+
+  def perform(inbox)
+    Current.executed_by = inbox.captain_assistant
+
+    resolvable_conversations = inbox.conversations.pending.where('last_activity_at < ? ', Time.now.utc - 1.hour).limit(Limits::BULK_ACTIONS_LIMIT)
+    resolvable_conversations.each do |conversation|
+      create_outgoing_message(conversation, inbox)
+      conversation.resolved!
+    end
+  ensure
+    Current.reset
+  end
+
+  private
+
+  def create_outgoing_message(conversation, inbox)
+    I18n.with_locale(inbox.account.locale) do
+      resolution_message = inbox.captain_assistant.config['resolution_message']
+      conversation.messages.create!(
+        {
+          message_type: :outgoing,
+          account_id: conversation.account_id,
+          inbox_id: conversation.inbox_id,
+          content: resolution_message.presence || I18n.t('conversations.activity.auto_resolution_message'),
+          sender: inbox.captain_assistant
+        }
+      )
+    end
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/llm/update_embedding_job.rb

@@ -0,0 +1,9 @@
+class Captain::Llm::UpdateEmbeddingJob < ApplicationJob
+  queue_as :low
+
+  def perform(record, content)
+    account_id = record.account_id
+    embedding = Captain::Llm::EmbeddingService.new(account_id: account_id).get_embedding(content)
+    record.update!(embedding: embedding)
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/tools/firecrawl_parser_job.rb

@@ -0,0 +1,28 @@
+class Captain::Tools::FirecrawlParserJob < ApplicationJob
+  queue_as :low
+
+  def perform(assistant_id:, payload:)
+    assistant = Captain::Assistant.find(assistant_id)
+    metadata = payload[:metadata]
+
+    canonical_url = normalize_link(metadata['url'])
+    document = assistant.documents.find_or_initialize_by(
+      external_link: canonical_url
+    )
+
+    document.update!(
+      external_link: canonical_url,
+      content: payload[:markdown],
+      name: metadata['title'],
+      status: :available
+    )
+  rescue StandardError => e
+    raise "Failed to parse FireCrawl data: #{e.message}"
+  end
+
+  private
+
+  def normalize_link(raw_url)
+    raw_url.to_s.delete_suffix('/')
+  end
+end

research/chatwoot/enterprise/app/jobs/captain/tools/simple_page_crawl_parser_job.rb

@@ -0,0 +1,39 @@
+class Captain::Tools::SimplePageCrawlParserJob < ApplicationJob
+  queue_as :low
+
+  def perform(assistant_id:, page_link:)
+    assistant = Captain::Assistant.find(assistant_id)
+    account = assistant.account
+
+    if limit_exceeded?(account)
+      Rails.logger.info("Document limit exceeded for #{assistant_id}")
+      return
+    end
+
+    crawler = Captain::Tools::SimplePageCrawlService.new(page_link)
+
+    page_title = crawler.page_title || ''
+    content = crawler.body_text_content || ''
+
+    normalized_link = normalize_link(page_link)
+    document = assistant.documents.find_or_initialize_by(external_link: normalized_link)
+
+    document.update!(
+      external_link: normalized_link,
+      name: page_title[0..254], content: content[0..14_999], status: :available
+    )
+  rescue StandardError => e
+    raise "Failed to parse data: #{page_link} #{e.message}"
+  end
+
+  private
+
+  def normalize_link(raw_link)
+    raw_link.to_s.delete_suffix('/')
+  end
+
+  def limit_exceeded?(account)
+    limits = account.usage_limits[:captain][:documents]
+    limits[:current_available].negative? || limits[:current_available].zero?
+  end
+end