[BLD] Switch to blacksmith runners (#4337)

## Description of changes

*Summarize the changes made by this PR.*
 - Improvements & Bug fixes
- This builds on #4312 but adds conditionality for windows runners on GH
 - New functionality
   - None

## Test plan
*How are these changes tested?*
These are tests
- [x] Tests pass locally with `pytest` for python, `yarn test` for js,
`cargo test` for rust

## Documentation Changes
None

---------

Co-authored-by: blacksmith-sh[bot] <157653362+blacksmith-sh[bot]@users.noreply.github.com>
Co-authored-by: Claude <[email protected]>

Author: 3 people

.github/actions/go/action.yaml

@@ -3,7 +3,15 @@ description: "This action sets up Go"
 runs:
   using: "composite"
   steps:
+    - uses: useblacksmith/setup-go@v6
+      # NOTE(hammadb) Blacksmith does not support windows, so we use the official action for windows and
+      # skip it on other platforms
+      if: runner.os != 'Windows'
+      with:
+        cache-dependency-path: go/go.sum
+
     - uses: actions/setup-go@v5
+      if: runner.os == 'Windows'
       with:
         cache-dependency-path: go/go.sum
 

.github/actions/python/action.yaml

@@ -8,12 +8,24 @@ inputs:
 runs:
   using: "composite"
   steps:
+
     - name: Set up Python 3.9 for protos
+      uses: useblacksmith/setup-python@v6
+      # NOTE(hammadb) Blacksmith does not support windows, so we use the official action for windows and
+      # skip it on other platforms
+      if: runner.os != 'Windows'
+      with:
+        python-version: "3.9"
+        cache: "pip"
+        cache-dependency-path: "requirements*.txt"
+    - name: Set up Python 3.9 for protos (Windows)
+      if: runner.os == 'Windows'
       uses: actions/setup-python@v5
       with:
         python-version: "3.9"
         cache: "pip"
         cache-dependency-path: "requirements*.txt"
+
     - name: Install proto dependencies
       run: |
         python -m pip install grpcio==1.58.0 grpcio-tools==1.58.0
@@ -31,6 +43,13 @@ runs:
         python -m pip uninstall -y grpcio grpcio-tools
       shell: bash
     - name: Set up Python ${{ inputs.python-version }}
+      uses: useblacksmith/setup-python@v6
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: "pip"
+        cache-dependency-path: "requirements*.txt"
+    - name: Set up Python ${{ inputs.python-version }} (Windows)
+      if: runner.os == 'Windows'
       uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}

.github/actions/rust/action.yaml

@@ -12,7 +12,7 @@ runs:
       # (reads from rust-toolchain.toml)
       run: |
         rustup --version
-        rustup toolchain install
+        rustup toolchain install 1.81.0
     # Needed for sccache to work on Windows
     - name: Set default toolchain to rust-toolchain.toml on Windows
       if: runner.os == 'Windows'

.github/actions/tilt/action.yaml

@@ -1,9 +1,5 @@
 name: Start Tilt services
 description: "This action starts Tilt services"
-inputs:
-  depot-project-id:
-    description: "Depot project ID"
-    required: true
 runs:
   using: "composite"
   steps:
@@ -25,8 +21,6 @@ runs:
     - name: Start Tilt
       shell: bash
       run: tilt ci
-      env:
-        DEPOT_PROJECT_ID: ${{ inputs.depot-project-id }}
     - name: Forward ports
       shell: bash
       run: |

.github/workflows/_build_js_bindings.yml

@@ -24,7 +24,7 @@ jobs:
           version: 9
           run_install: false
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: useblacksmith/setup-node@v5
         with:
           node-version: 20
           cache: pnpm
@@ -72,7 +72,7 @@ jobs:
           name: bindings-x86_64-apple-darwin
           path: rust/js_bindings/chromadb-js-bindings.darwin-x64.node
           if-no-files-found: error
-          
+
   build-windows:
     name: Build Windows bindings
     runs-on: windows-latest
@@ -87,7 +87,7 @@ jobs:
           version: 9
           run_install: false
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: useblacksmith/setup-node@v5
         with:
           node-version: 20
           cache: pnpm
@@ -125,10 +125,10 @@ jobs:
           name: bindings-x86_64-pc-windows-msvc
           path: rust/js_bindings/chromadb-js-bindings.win32-x64-msvc.node
           if-no-files-found: error
-          
+
   build-linux:
     name: Build Linux bindings
-    runs-on: depot-ubuntu-22.04-16
+    runs-on: blacksmith-16vcpu-ubuntu-2204
     defaults:
       run:
         working-directory: rust/js_bindings
@@ -140,7 +140,7 @@ jobs:
           version: 9
           run_install: false
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: useblacksmith/setup-node@v5
         with:
           node-version: 20
           cache: pnpm
@@ -221,7 +221,7 @@ jobs:
           version: 9
           run_install: false
       - name: Setup node
-        uses: actions/setup-node@v4
+        uses: useblacksmith/setup-node@v5
         with:
           node-version: 20
           cache: pnpm
@@ -241,12 +241,12 @@ jobs:
         run: |
           set -e
           npm config set provenance true
-          echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc          
+          echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
           for dir in npm/*; do
             if [ -d "$dir" ]; then
               cd "$dir" && npm publish --access public && cd -
             fi
           done
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/_build_release_container.yml

@@ -38,7 +38,7 @@ env:
 jobs:
   build:
     name: Build and publish container image
-    runs-on: depot-ubuntu-22.04
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -51,8 +51,6 @@ jobs:
           ghcr-password: ${{ secrets.GITHUB_TOKEN }}
           dockerhub-username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub-password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Set up Depot CLI
-        uses: depot/setup-action@v1
       - name: Compute tags
         shell: bash
         id: compute_tags
@@ -65,7 +63,7 @@ jobs:
           echo "tags=${tags}" >> $GITHUB_OUTPUT
 
       - name: Build and push image
-        uses: depot/build-push-action@v1
+        uses: useblacksmith/build-push-action@v1.1
         with:
           context: .
           file: rust/cli/Dockerfile
@@ -74,4 +72,3 @@ jobs:
           build-args: |
             RELEASE_MODE=1
           tags: ${{ steps.compute_tags.outputs.tags }}
-          project: ${{ vars.DEPOT_PROJECT_ID }}

.github/workflows/_build_release_pypi.yml

@@ -41,7 +41,7 @@ permissions:
 jobs:
   version:
     name: Resolve version
-    runs-on: depot-ubuntu-22.04
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     outputs:
       version: ${{ steps.resolve_version.outputs.version }}
     steps:
@@ -65,11 +65,11 @@ jobs:
       fail-fast: false
       matrix:
         platform:
-          - { os: linux, runner: depot-ubuntu-22.04, target: x86_64 }
-          - { os: linux, runner: depot-ubuntu-22.04-arm, target: aarch64 }
+          - { os: linux, runner: blacksmith-4vcpu-ubuntu-2204, target: x86_64 }
+          - { os: linux, runner: blacksmith-4vcpu-ubuntu-2204-arm, target: aarch64 }
           - { os: windows, runner: windows-latest, target: x64 }
-          - { os: macos, runner: depot-macos-14, target: x86_64 }
-          - { os: macos, runner: depot-macos-14, target: aarch64 }
+          - { os: macos, runner: macos-14, target: x86_64 }
+          - { os: macos, runner: macos-14, target: aarch64 }
 
     steps:
       - uses: actions/checkout@v4
@@ -81,14 +81,15 @@ jobs:
       # On Windows, we want to set up Python with the correct architecture
       - name: Set up Python on Windows
         if: ${{ matrix.platform.os == 'windows' }}
+        # Blacksmith does not currently support windows, so we use the official action
         uses: actions/setup-python@v5
         with:
           python-version: 3.x
           architecture: ${{ matrix.platform.target }}
 
       - name: Set up Python on non-Windows
         if: ${{ matrix.platform.os != 'windows' }}
-        uses: actions/setup-python@v5
+        uses: useblacksmith/setup-python@v6
         with:
           python-version: 3.x
 
@@ -127,7 +128,7 @@ jobs:
 
   sdist:
     name: build-sdist
-    runs-on: depot-ubuntu-22.04
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     needs: version
     steps:
       - uses: actions/checkout@v4
@@ -170,7 +171,7 @@ jobs:
 
   release:
     name: Release
-    runs-on: ubuntu-22.04
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     if: ${{ inputs.publish_to_pypi || inputs.publish_to_test_pypi }}
     needs: [build, sdist]
     permissions:

.github/workflows/_go-tests.yml

@@ -5,8 +5,8 @@ on:
 
 jobs:
   cluster-test:
-    runs-on: "depot-ubuntu-22.04-16"
-    # OIDC token auth for Depot & AWS
+    runs-on: "blacksmith-16vcpu-ubuntu-2204"
+    # OIDC token auth for AWS
     permissions:
       contents: read
       id-token: write
@@ -15,9 +15,10 @@ jobs:
         uses: actions/checkout@v4
       - name: Setup
         uses: ./.github/actions/go
-      - uses: ./.github/actions/tilt
+      - uses: useblacksmith/[email protected]
         with:
-          depot-project-id: ${{ vars.DEPOT_PROJECT_ID }}
+          setup-only: true
+      - uses: ./.github/actions/tilt
       - run: bin/cluster-test.sh bash -c 'cd go && make test'
       - name: Save service logs to artifact
         if: always()

.github/workflows/_javascript-client-tests.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   test:
-    runs-on: depot-ubuntu-22.04
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/_python-tests.yml

@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         python: ${{fromJson(inputs.python_versions)}}
-        platform: [depot-ubuntu-24.04-8, depot-windows-2022-8]
+        platform: [blacksmith-8vcpu-ubuntu-2204, 16core-64gb-windows-latest]
         test-globs:
           - "--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore-glob 'chromadb/test/distributed/*'"
           - "chromadb/test/property --ignore-glob chromadb/test/property/test_cross_version_persist.py"
@@ -70,9 +70,7 @@ jobs:
       fail-fast: false
       matrix:
         python: ${{fromJson(inputs.python_versions)}}
-        # NOTE(hammadb): For some reason depot-windows fails so leaving default windows-latest
-        # on chromadb/test/client/test_multiple_clients_concurrency.py::test_multiple_clients_concurrently
-        platform: [depot-ubuntu-24.04, windows-latest]
+        platform: [blacksmith-4vcpu-ubuntu-2204, windows-latest]
         test-globs: ["--ignore-glob 'chromadb/test/property/*' --ignore-glob 'chromadb/test/stress/*' --ignore='chromadb/test/test_cli.py' --ignore-glob 'chromadb/test/distributed/*'",
                     "chromadb/test/property/test_add.py",
                     "chromadb/test/property/test_collections.py",
@@ -83,7 +81,7 @@ jobs:
                     "chromadb/test/property/test_persist.py",
                     "chromadb/test/stress"]
         include:
-          - platform: depot-ubuntu-24.04
+          - platform: blacksmith-4vcpu-ubuntu-2204
             env-file: compose-env.linux
     runs-on: ${{ matrix.platform }}
     steps:
@@ -114,7 +112,7 @@ jobs:
     strategy:
       matrix:
         python: ${{fromJson(inputs.python_versions)}}
-        platform: [depot-ubuntu-24.04, depot-windows-2022]
+        platform: [blacksmith-4vcpu-ubuntu-2204, windows-latest]
         test-globs: ["chromadb/test/property/test_add.py",
                     "chromadb/test/property/test_collections.py",
                     "chromadb/test/property/test_collections_with_database_tenant.py",
@@ -152,7 +150,7 @@ jobs:
       fail-fast: false
       matrix:
         python: ${{fromJson(inputs.python_versions)}}
-        platform: ["depot-ubuntu-22.04-4"]
+        platform: ["blacksmith-16vcpu-ubuntu-2204"]
         test-globs: ["chromadb/test/api",
                     "chromadb/test/api/test_collection.py",
                     "chromadb/test/api/test_limit_offset.py",
@@ -164,7 +162,7 @@ jobs:
                     "chromadb/test/property/test_collections_with_database_tenant_overwrite.py",
                     "chromadb/test/distributed/test_sanity.py"]
     runs-on: ${{ matrix.platform }}
-    # OIDC token auth for Depot & AWS
+    # OIDC token auth for AWS
     permissions:
       contents: read
       id-token: write
@@ -173,15 +171,17 @@ jobs:
       - uses: ./.github/actions/python
         with:
           python-version: ${{ matrix.python }}
-      - name: Configure pytest to upload results to Datadog
-        uses: datadog/test-visibility-github-action@v2
+      # TODO(adityamaru): Add Datadog test visibility when running in Chroma's repo.
+      # - name: Configure pytest to upload results to Datadog
+      #   uses: datadog/test-visibility-github-action@v2
+      #   with:
+      #     languages: python
+      #     api_key: ${{ secrets.DD_API_KEY }}
+      #     site: ${{ vars.DD_SITE }}
+      - uses: useblacksmith/[email protected]
         with:
-          languages: python
-          api_key: ${{ secrets.DD_API_KEY }}
-          site: ${{ vars.DD_SITE }}
+          setup-only: true
       - uses: ./.github/actions/tilt
-        with:
-          depot-project-id: ${{ vars.DEPOT_PROJECT_ID }}
       - name: Test
         run: bin/cluster-test.sh bash -c 'python -m pytest "${{ matrix.test-globs }}"' --durations 10
         shell: bash
@@ -200,7 +200,7 @@ jobs:
           artifact-name: ${{ steps.compute-artifact-name.outputs.artifact_name }}
 
   merge-cluster-logs:
-    runs-on: ubuntu-latest
+    runs-on: blacksmith-4vcpu-ubuntu-2204
     needs: test-cluster-rust-frontend
     steps:
       - name: Merge
@@ -215,7 +215,7 @@ jobs:
       fail-fast: false
       matrix:
         python: ${{fromJson(inputs.python_versions)}}
-        platform: [depot-ubuntu-24.04-16, depot-windows-2022-16]
+        platform: [blacksmith-16vcpu-ubuntu-2204, 16core-64gb-windows-latest]
         test-globs: ["chromadb/test/stress"]
     runs-on: ${{ matrix.platform }}
     steps:
@@ -256,10 +256,10 @@ jobs:
       fail-fast: false
       matrix:
         python: ${{fromJson(inputs.python_versions)}}
-        platform: [depot-ubuntu-24.04, windows-latest]
+        platform: [blacksmith-4vcpu-ubuntu-2204, windows-latest]
         test-globs: ["chromadb/test/test_cli.py"]
         include:
-          - platform: depot-ubuntu-24.04
+          - platform: blacksmith-4vcpu-ubuntu-2204
             env-file: compose-env.linux
           - platform: windows-latest
             env-file: compose-env.windows