Restructure omni services and add Chatwoot research snapshot

research/chatwoot/.circleci/config.yml

@@ -0,0 +1,374 @@
+version: 2.1
+orbs:
+  node: circleci/node@6.1.0
+  qlty-orb: qltysh/qlty-orb@0.0
+
+# Shared defaults for setup steps
+defaults: &defaults
+  working_directory: ~/build
+  machine:
+    image: ubuntu-2204:2024.05.1
+  resource_class: large
+  environment:
+    RAILS_LOG_TO_STDOUT: false
+    COVERAGE: true
+    LOG_LEVEL: warn
+
+jobs:
+  # Separate job for linting (no parallelism needed)
+  lint:
+    <<: *defaults
+    steps:
+      - checkout
+
+      # Install minimal system dependencies for linting
+      - run:
+          name: Install System Dependencies
+          command: |
+            sudo apt-get update
+            DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \
+              libpq-dev \
+              build-essential \
+              git \
+              curl \
+              libssl-dev \
+              zlib1g-dev \
+              libreadline-dev \
+              libyaml-dev \
+              openjdk-11-jdk \
+              jq \
+              software-properties-common \
+              ca-certificates \
+              imagemagick \
+              libxml2-dev \
+              libxslt1-dev \
+              file \
+              g++ \
+              gcc \
+              autoconf \
+              gnupg2 \
+              patch \
+              ruby-dev \
+              liblzma-dev \
+              libgmp-dev \
+              libncurses5-dev \
+              libffi-dev \
+              libgdbm6 \
+              libgdbm-dev \
+              libvips
+
+      - run:
+          name: Install RVM and Ruby 3.4.4
+          command: |
+            sudo apt-get install -y gpg
+            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+            \curl -sSL https://get.rvm.io | bash -s stable
+            echo 'source ~/.rvm/scripts/rvm' >> $BASH_ENV
+            source ~/.rvm/scripts/rvm
+            rvm install "3.4.4"
+            rvm use 3.4.4 --default
+            gem install bundler -v 2.5.16
+
+      - run:
+          name: Install Application Dependencies
+          command: |
+            source ~/.rvm/scripts/rvm
+            bundle install
+
+      - node/install:
+          node-version: '24.13'
+      - node/install-pnpm
+      - node/install-packages:
+          pkg-manager: pnpm
+          override-ci-command: pnpm i
+
+      # Swagger verification
+      - run:
+          name: Verify swagger API specification
+          command: |
+            bundle exec rake swagger:build
+            if [[ `git status swagger/swagger.json --porcelain` ]]
+            then
+              echo "ERROR: The swagger.json file is not in sync with the yaml specification. Run 'rake swagger:build' and commit 'swagger/swagger.json'."
+              exit 1
+            fi
+            mkdir -p ~/tmp
+            curl -L https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.3.0/openapi-generator-cli-6.3.0.jar > ~/tmp/openapi-generator-cli-6.3.0.jar
+            java -jar ~/tmp/openapi-generator-cli-6.3.0.jar validate -i swagger/swagger.json
+
+      # Bundle audit
+      - run:
+          name: Bundle audit
+          command: bundle exec bundle audit update && bundle exec bundle audit check -v
+
+      # Rubocop linting
+      - run:
+          name: Rubocop
+          command: bundle exec rubocop --parallel
+
+      # ESLint linting
+      - run:
+          name: eslint
+          command: pnpm run eslint
+
+  # Separate job for frontend tests
+  frontend-tests:
+    <<: *defaults
+    steps:
+      - checkout
+      - node/install:
+          node-version: '24.13'
+      - node/install-pnpm
+      - node/install-packages:
+          pkg-manager: pnpm
+          override-ci-command: pnpm i
+
+      - run:
+          name: Run frontend tests (with coverage)
+          command: pnpm run test:coverage
+
+      - run:
+          name: Move coverage files if they exist
+          command: |
+            if [ -d "coverage" ]; then
+              mkdir -p ~/build/coverage
+              cp -r coverage ~/build/coverage/frontend || true
+            fi
+          when: always
+
+      - persist_to_workspace:
+          root: ~/build
+          paths:
+            - coverage
+
+  # Backend tests with parallelization
+  backend-tests:
+    <<: *defaults
+    parallelism: 18
+    steps:
+      - checkout
+      - node/install:
+          node-version: '24.13'
+      - node/install-pnpm
+      - node/install-packages:
+          pkg-manager: pnpm
+          override-ci-command: pnpm i
+
+      - run:
+          name: Add PostgreSQL repository and update
+          command: |
+            sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+            wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+            sudo apt-get update -y
+
+      - run:
+          name: Install System Dependencies
+          command: |
+            sudo apt-get update
+            DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \
+              libpq-dev \
+              redis-server \
+              postgresql-common \
+              postgresql-16 \
+              postgresql-16-pgvector \
+              build-essential \
+              git \
+              curl \
+              libssl-dev \
+              zlib1g-dev \
+              libreadline-dev \
+              libyaml-dev \
+              openjdk-11-jdk \
+              jq \
+              software-properties-common \
+              ca-certificates \
+              imagemagick \
+              libxml2-dev \
+              libxslt1-dev \
+              file \
+              g++ \
+              gcc \
+              autoconf \
+              gnupg2 \
+              patch \
+              ruby-dev \
+              liblzma-dev \
+              libgmp-dev \
+              libncurses5-dev \
+              libffi-dev \
+              libgdbm6 \
+              libgdbm-dev \
+              libvips
+
+      - run:
+          name: Install RVM and Ruby 3.4.4
+          command: |
+            sudo apt-get install -y gpg
+            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+            \curl -sSL https://get.rvm.io | bash -s stable
+            echo 'source ~/.rvm/scripts/rvm' >> $BASH_ENV
+            source ~/.rvm/scripts/rvm
+            rvm install "3.4.4"
+            rvm use 3.4.4 --default
+            gem install bundler -v 2.5.16
+
+      - run:
+          name: Install Application Dependencies
+          command: |
+            source ~/.rvm/scripts/rvm
+            bundle install
+
+      # Install and configure OpenSearch
+      - run:
+          name: Install OpenSearch
+          command: |
+            # Download and install OpenSearch 2.11.0 (compatible with Elasticsearch 7.x clients)
+            wget https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.0/opensearch-2.11.0-linux-x64.tar.gz
+            tar -xzf opensearch-2.11.0-linux-x64.tar.gz
+            sudo mv opensearch-2.11.0 /opt/opensearch
+
+      - run:
+          name: Configure and Start OpenSearch
+          command: |
+            # Configure OpenSearch for single-node testing
+            cat > /opt/opensearch/config/opensearch.yml \<< EOF
+            cluster.name: chatwoot-test
+            node.name: node-1
+            network.host: 0.0.0.0
+            http.port: 9200
+            discovery.type: single-node
+            plugins.security.disabled: true
+            EOF
+
+            # Set ownership and permissions
+            sudo chown -R $USER:$USER /opt/opensearch
+
+            # Start OpenSearch in background
+            /opt/opensearch/bin/opensearch -d -p /tmp/opensearch.pid
+
+      - run:
+          name: Wait for OpenSearch to be ready
+          command: |
+            echo "Waiting for OpenSearch to start..."
+            for i in {1..30}; do
+              if curl -s http://localhost:9200/_cluster/health | grep -q '"status"'; then
+                echo "OpenSearch is ready!"
+                exit 0
+              fi
+              echo "Waiting... ($i/30)"
+              sleep 2
+            done
+            echo "OpenSearch failed to start"
+            exit 1
+
+      # Configure environment and database
+      - run:
+          name: Database Setup and Configure Environment Variables
+          command: |
+            pg_pass=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 15 ; echo '')
+            sed -i "s/REPLACE_WITH_PASSWORD/${pg_pass}/g" ${PWD}/.circleci/setup_chatwoot.sql
+            chmod 644 ${PWD}/.circleci/setup_chatwoot.sql
+            mv ${PWD}/.circleci/setup_chatwoot.sql /tmp/
+            sudo -i -u postgres psql -f /tmp/setup_chatwoot.sql
+            cp .env.example .env
+            sed -i '/^FRONTEND_URL/d' .env
+            sed -i -e '/REDIS_URL/ s/=.*/=redis:\/\/localhost:6379/' .env
+            sed -i -e '/POSTGRES_HOST/ s/=.*/=localhost/' .env
+            sed -i -e '/POSTGRES_USERNAME/ s/=.*/=chatwoot/' .env
+            sed -i -e "/POSTGRES_PASSWORD/ s/=.*/=$pg_pass/" .env
+            echo -en "\nINSTALLATION_ENV=circleci" >> ".env"
+            echo -en "\nOPENSEARCH_URL=http://localhost:9200" >> ".env"
+
+      # Database setup
+      - run:
+          name: Run DB migrations
+          command: bundle exec rails db:chatwoot_prepare
+
+      # Run backend tests (parallelized)
+      - run:
+          name: Run backend tests
+          command: |
+            mkdir -p ~/tmp/test-results/rspec
+            mkdir -p ~/tmp/test-artifacts
+            mkdir -p ~/build/coverage/backend
+
+            # Use round-robin distribution (same as GitHub Actions) for better test isolation
+            # This prevents tests with similar timing from being grouped on the same runner
+            SPEC_FILES=($(find spec -name '*_spec.rb' | sort))
+            TESTS=""
+
+            for i in "${!SPEC_FILES[@]}"; do
+              if [ $(( i % $CIRCLE_NODE_TOTAL )) -eq $CIRCLE_NODE_INDEX ]; then
+                TESTS="$TESTS ${SPEC_FILES[$i]}"
+              fi
+            done
+
+            bundle exec rspec -I ./spec --require coverage_helper --require spec_helper --format progress \
+                              --format RspecJunitFormatter \
+                              --out ~/tmp/test-results/rspec.xml \
+                              -- $TESTS
+          no_output_timeout: 30m
+
+      # Store test results for better splitting in future runs
+      - store_test_results:
+          path: ~/tmp/test-results
+
+      - run:
+          name: Move coverage files if they exist
+          command: |
+            if [ -d "coverage" ]; then
+              mkdir -p ~/build/coverage
+              cp -r coverage ~/build/coverage/backend || true
+            fi
+          when: always
+
+      - persist_to_workspace:
+          root: ~/build
+          paths:
+            - coverage
+
+  # Collect coverage from all jobs
+  coverage:
+    <<: *defaults
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/build
+
+      # Qlty coverage publish
+      - qlty-orb/coverage_publish:
+          files: |
+            coverage/frontend/lcov.info
+
+      - run:
+          name: List coverage directory contents
+          command: |
+            ls -R ~/build/coverage || echo "No coverage directory"
+
+      - store_artifacts:
+          path: coverage
+          destination: coverage
+
+  build:
+    <<: *defaults
+    steps:
+      - run:
+          name: Legacy build aggregator
+          command: |
+            echo "All main jobs passed; build job kept only for GitHub required check compatibility."
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - lint
+      - frontend-tests
+      - backend-tests
+      - coverage:
+          requires:
+            - frontend-tests
+            - backend-tests
+      - build:
+          requires:
+            - lint
+            - coverage