_python-tests.yml

name: Chroma Python Base Tests

on:
  workflow_call:
    inputs:
      python_versions:
        description: 'Python versions to test (as json array)'
        required: false
        default: '["3.9"]'
        type: string
      property_testing_preset:
        description: 'Property testing preset'
        required: true
        type: string

jobs:
  test-rust-bindings:
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        python: ${{fromJson(inputs.python_versions)}}
        platform: [blacksmith-8vcpu-ubuntu-2204, 16core-64gb-windows-latest]
        test-globs:
          - "--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore-glob 'chromadb/test/distributed/*'"
          - "chromadb/test/property --ignore-glob chromadb/test/property/test_cross_version_persist.py"
          # TODO(@codetheweb): this test should be parallelized, but it frequently flakes in CI when parallelized. We have not been able to reproduce the failure locally. See https://github.com/chroma-core/chroma/issues/4263.
          - "chromadb/test/property/test_cross_version_persist.py"
        include:
          - test-globs: "chromadb/test/property --ignore-glob chromadb/test/property/test_cross_version_persist.py"
            parallelized: true

    runs-on: ${{ matrix.platform }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: ./.github/actions/python
        with:
          python-version: ${{ matrix.python }}
      - name: Setup Rust
        uses: ./.github/actions/rust
        with:
          github-token: ${{ github.token }}
      - name: Build Rust bindings
        uses: PyO3/maturin-action@v1
        with:
          command: build
          sccache: true
      - name: Install built wheel
        shell: bash
        run: pip install --no-index --find-links target/wheels/ chromadb
      - name: Configure pytest to upload results to Datadog
        uses: datadog/test-visibility-github-action@v2
        # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
        if: ${{ !contains(matrix.platform, 'windows') }}
        with:
          languages: python
          api_key: ${{ secrets.DD_API_KEY }}
          site: ${{ vars.DD_SITE }}
      - name: Test
        run: python -m pytest ${{ matrix.test-globs }} ${{ matrix.parallelized && '-n auto --dist worksteal' || '' }} -v --color=yes --durations 10
        shell: bash
        env:
          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
          CHROMA_RUST_BINDINGS_TEST_ONLY: "1"
          RUST_BACKTRACE: 1

  test-rust-single-node-integration:
    strategy:
      fail-fast: false
      matrix:
        python: ${{fromJson(inputs.python_versions)}}
        platform: [blacksmith-4vcpu-ubuntu-2204, windows-latest]
        test-globs: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore='chromadb/test/test_cli.py' --ignore-glob 'chromadb/test/distributed/*'",
                    "chromadb/test/property/test_add.py",
                    "chromadb/test/property/test_collections.py",
                    "chromadb/test/property/test_collections_with_database_tenant.py",
                    "chromadb/test/property/test_cross_version_persist.py",
                    "chromadb/test/property/test_embeddings.py",
                    "chromadb/test/property/test_filtering.py",
                    "chromadb/test/property/test_persist.py",
                    "chromadb/test/stress"]
        include:
          - platform: blacksmith-4vcpu-ubuntu-2204
            env-file: compose-env.linux
    runs-on: ${{ matrix.platform }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Set up Python (${{ matrix.python }})
      uses: ./.github/actions/python
    - name: Setup Rust
      uses: ./.github/actions/rust
      with:
          github-token: ${{ github.token }}
    - name: Configure pytest to upload results to Datadog
      uses: datadog/test-visibility-github-action@v2
      # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
      if: ${{ !contains(matrix.platform, 'windows') }}
      with:
        languages: python
        api_key: ${{ secrets.DD_API_KEY }}
        site: ${{ vars.DD_SITE }}
    - name: Rust Integration Test
      run: bin/rust-integration-test.sh ${{ matrix.test-globs }}
      shell: bash
      env:
        ENV_FILE: ${{ matrix.env-file }}
        PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

  test-rust-thin-client:
    strategy:
      matrix:
        python: ${{fromJson(inputs.python_versions)}}
        platform: [blacksmith-4vcpu-ubuntu-2204, windows-latest]
        test-globs: ["chromadb/test/property/test_add.py",
                    "chromadb/test/property/test_collections.py",
                    "chromadb/test/property/test_collections_with_database_tenant.py",
                    "chromadb/test/property/test_embeddings.py",
                    "chromadb/test/property/test_filtering.py"]
    runs-on: ${{ matrix.platform }}
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python (${{ matrix.python }})
        uses: ./.github/actions/python
        with:
          python-version: ${{ matrix.python }}
      - name: Setup Rust
        uses: ./.github/actions/rust
        with:
          github-token: ${{ github.token }}
      - name: Configure pytest to upload results to Datadog
        uses: datadog/test-visibility-github-action@v2
        # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
        if: ${{ !contains(matrix.platform, 'windows') }}
        with:
          languages: python
          api_key: ${{ secrets.DD_API_KEY }}
          site: ${{ vars.DD_SITE }}
      - name: Test
        run: bin/rust-integration-test.sh ${{ matrix.test-globs }}
        shell: bash
        env:
          CHROMA_THIN_CLIENT: "1"
          ENV_FILE: ${{ matrix.env-file }}
          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}

  test-cluster-rust-frontend:
    strategy:
      fail-fast: false
      matrix:
        python: ${{fromJson(inputs.python_versions)}}
        platform: ["blacksmith-16vcpu-ubuntu-2204"]
        test-globs: ["chromadb/test/api",
                    "chromadb/test/api/test_collection.py",
                    "chromadb/test/api/test_limit_offset.py",
                    "chromadb/test/property/test_collections.py",
                    "chromadb/test/property/test_add.py",
                    "chromadb/test/property/test_filtering.py",
                    "chromadb/test/property/test_fork.py",
                    "chromadb/test/property/test_embeddings.py",
                    "chromadb/test/property/test_collections_with_database_tenant.py",
                    "chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
                    "chromadb/test/distributed/test_sanity.py"]
    runs-on: ${{ matrix.platform }}
    # OIDC token auth for AWS
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/python
        with:
          python-version: ${{ matrix.python }}
      # TODO(adityamaru): Add Datadog test visibility when running in Chroma's repo.
      # - name: Configure pytest to upload results to Datadog
      #   uses: datadog/test-visibility-github-action@v2
      #   with:
      #     languages: python
      #     api_key: ${{ secrets.DD_API_KEY }}
      #     site: ${{ vars.DD_SITE }}
      - uses: useblacksmith/build-push-action@v1.1
        with:
          setup-only: true
      - uses: ./.github/actions/tilt
      - name: Test
        run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}"' --durations 10
        shell: bash
        env:
          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
          CHROMA_RUST_FRONTEND_TEST_ONLY: "1"
          CHROMA_SERVER_HOST: "localhost:3000"
      - name: Compute artifact name
        if: always()
        id: compute-artifact-name
        run: echo "artifact_name=cluster_logs_rust_frontend_$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}" >> $GITHUB_OUTPUT
      - name: Save service logs to artifact
        if: always()
        uses: ./.github/actions/export-tilt-logs
        with:
          artifact-name: ${{ steps.compute-artifact-name.outputs.artifact_name }}

  merge-cluster-logs:
    runs-on: blacksmith-4vcpu-ubuntu-2204
    needs: test-cluster-rust-frontend
    steps:
      - name: Merge
        uses: actions/upload-artifact/merge@v4
        with:
          name: cluster_test_logs
          pattern: cluster_logs_*

  test-rust-bindings-stress:
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        python: ${{fromJson(inputs.python_versions)}}
        platform: [blacksmith-16vcpu-ubuntu-2204, 16core-64gb-windows-latest]
        test-globs: ["chromadb/test/stress"]
    runs-on: ${{ matrix.platform }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Python
        uses: ./.github/actions/python
        with:
          python-version: ${{ matrix.python }}
      - name: Setup Rust
        uses: ./.github/actions/rust
        with:
          github-token: ${{ github.token }}
      - name: Build Rust bindings
        uses: PyO3/maturin-action@v1
        with:
          command: build
          sccache: true
      - name: Install built wheel
        shell: bash
        run: pip install --no-index --find-links target/wheels/ chromadb
      - name: Configure pytest to upload results to Datadog
        uses: datadog/test-visibility-github-action@v2
        # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
        if: ${{ !contains(matrix.platform, 'windows') }}
        with:
          languages: python
          api_key: ${{ secrets.DD_API_KEY }}
          site: ${{ vars.DD_SITE }}
      - name: Test
        run: python -m pytest ${{ matrix.test-globs }} --durations 10
        shell: bash
        env:
          PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
          CHROMA_RUST_BINDINGS_TEST_ONLY: "1"

  test-python-cli:
    strategy:
      fail-fast: false
      matrix:
        python: ${{fromJson(inputs.python_versions)}}
        platform: [blacksmith-4vcpu-ubuntu-2204, windows-latest]
        test-globs: ["chromadb/test/test_cli.py"]
        include:
          - platform: blacksmith-4vcpu-ubuntu-2204
            env-file: compose-env.linux
          - platform: windows-latest
            env-file: compose-env.windows
    runs-on: ${{ matrix.platform }}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Set up Python (${{ matrix.python }})
      uses: ./.github/actions/python
    - name: Setup Rust
      uses: ./.github/actions/rust
      with:
        github-token: ${{ github.token }}
    - name: Build Rust bindings
      uses: PyO3/maturin-action@v1
      with:
        command: build
        sccache: true
    - name: Install built wheel
      shell: bash
      run: pip install --no-index --find-links target/wheels/ chromadb
    - name: Configure pytest to upload results to Datadog
      uses: datadog/test-visibility-github-action@v2
      # This action currently fails on Windows (https://github.com/DataDog/test-visibility-github-action/issues/36)
      if: ${{ !contains(matrix.platform, 'windows') }}
      with:
        languages: python
        api_key: ${{ secrets.DD_API_KEY }}
        site: ${{ vars.DD_SITE }}
    - name: Integration Test
      run: bin/python-integration-test ${{ matrix.test-globs }}
      shell: bash
      env:
        ENV_FILE: ${{ matrix.env-file }}
        PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}