gha-e2e-tests.yaml

name: (gha) E2E Tests

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  workflow_dispatch:

permissions:
  contents: read

env:
  TARGET_ORG: actions-runner-controller
  TARGET_REPO: arc_e2e_test_dummy
  IMAGE_NAME: "arc-test-image"
  IMAGE_VERSION: "0.11.0"

concurrency:
  # This will make sure we only apply the concurrency limits on pull requests
  # but not pushes to master branch by making the concurrency group name unique
  # for pushes
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  default-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-workflow.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  single-namespace-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-workflow.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          kubectl create namespace arc-runners
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          --set flags.watchSingleNamespace=arc-runners \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  dind-mode-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: arc-test-dind-workflow.yaml
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          --set containerMode.type="dind" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  kubernetes-mode-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-kubernetes-workflow.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          echo "Install openebs/dynamic-localpv-provisioner"
          helm repo add openebs https://openebs.github.io/charts
          helm repo update
          helm install openebs openebs/openebs -n openebs --create-namespace

          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems
          kubectl wait --timeout=30s --for=condition=ready pod -n openebs -l name=openebs-localpv-provisioner

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          --set containerMode.type="kubernetes" \
          --set containerMode.kubernetesModeWorkVolumeClaim.accessModes={"ReadWriteOnce"} \
          --set containerMode.kubernetesModeWorkVolumeClaim.storageClassName="openebs-hostpath" \
          --set containerMode.kubernetesModeWorkVolumeClaim.resources.requests.storage="1Gi" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  auth-proxy-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-workflow.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          docker run -d \
            --name squid \
            --publish 3128:3128 \
            huangtingluo/squid-proxy:latest
          kubectl create namespace arc-runners
          kubectl create secret generic proxy-auth \
            --namespace=arc-runners \
            --from-literal=username=github \
            --from-literal=password='actions'
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          --set proxy.https.url="http://host.minikube.internal:3128" \
          --set proxy.https.credentialSecretRef="proxy-auth" \
          --set "proxy.noProxy[0]=10.96.0.1:443" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  anonymous-proxy-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-workflow.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          docker run -d \
            --name squid \
            --publish 3128:3128 \
            ubuntu/squid:latest
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          --set proxy.https.url="http://host.minikube.internal:3128" \
          --set "proxy.noProxy[0]=10.96.0.1:443" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  self-signed-ca-setup:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-workflow.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          docker run -d \
            --rm \
            --name mitmproxy \
            --publish 8080:8080 \
            -v ${{ github.workspace }}/mitmproxy:/home/mitmproxy/.mitmproxy \
            mitmproxy/mitmproxy:latest \
            mitmdump
          count=0
          while true; do
            if [ -f "${{ github.workspace }}/mitmproxy/mitmproxy-ca-cert.pem" ]; then
              echo "CA cert generated"
              cat ${{ github.workspace }}/mitmproxy/mitmproxy-ca-cert.pem
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for mitmproxy generate its CA cert"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          sudo cp ${{ github.workspace }}/mitmproxy/mitmproxy-ca-cert.pem ${{ github.workspace }}/mitmproxy/mitmproxy-ca-cert.crt
          sudo chown runner ${{ github.workspace }}/mitmproxy/mitmproxy-ca-cert.crt
          kubectl create namespace arc-runners
          kubectl -n arc-runners create configmap ca-cert --from-file="${{ github.workspace }}/mitmproxy/mitmproxy-ca-cert.crt"
          kubectl -n arc-runners get configmap ca-cert -o yaml
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          --set proxy.https.url="http://host.minikube.internal:8080" \
          --set "proxy.noProxy[0]=10.96.0.1:443" \
          --set "githubServerTLS.certificateFrom.configMapKeyRef.name=ca-cert" \
          --set "githubServerTLS.certificateFrom.configMapKeyRef.key=mitmproxy-ca-cert.crt" \
          --set "githubServerTLS.runnerMountPath=/usr/local/share/ca-certificates/" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Test ARC E2E
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"

  update-strategy-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: "arc-test-sleepy-matrix.yaml"
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{github.head_ref}}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          --set flags.updateStrategy="eventual" \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems

          sleep 60

      - name: Trigger long running jobs and wait for runners to pick them up
        uses: ./.github/actions/execute-assert-arc-e2e
        timeout-minutes: 10
        with:
          auth-token: ${{ steps.setup.outputs.token }}
          repo-owner: ${{ env.TARGET_ORG }}
          repo-name: ${{env.TARGET_REPO}}
          workflow-file: ${{env.WORKFLOW_FILE}}
          arc-name: ${{steps.install_arc.outputs.ARC_NAME}}
          arc-namespace: "arc-runners"
          arc-controller-namespace: "arc-systems"
          wait-to-running: "true"
          wait-to-finish: "false"

      - name: Upgrade the gha-runner-scale-set
        shell: bash
        run: |
          helm upgrade --install "${{ steps.install_arc.outputs.ARC_NAME }}" \
            --namespace "arc-runners" \
            --create-namespace \
            --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{ env.TARGET_REPO }}" \
            --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
            --set template.spec.containers[0].name="runner" \
            --set template.spec.containers[0].image="ghcr.io/actions/actions-runner:latest" \
            --set template.spec.containers[0].command={"/home/runner/run.sh"} \
            --set template.spec.containers[0].env[0].name="TEST" \
            --set template.spec.containers[0].env[0].value="E2E TESTS" \
            ./charts/gha-runner-scale-set \
            --debug

      - name: Assert that the listener is deleted while jobs are running
        shell: bash
        run: |
          count=0
          while true; do
            LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name=${{ steps.install_arc.outputs.ARC_NAME }} -n arc-systems --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
            RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n arc-runners --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
            RESOURCES="$(kubectl get pods -A)"

            if [ "$LISTENER_COUNT" -eq 0 ]; then
              echo "Listener has been deleted"
              echo "$RESOURCES"
              exit 0
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener to be deleted"
              echo "$RESOURCES"
              exit 1
            fi

            echo "Waiting for listener to be deleted"
            echo "Listener count: $LISTENER_COUNT target: 0 | Runners count: $RUNNERS_COUNT target: 3"

            sleep 1
            count=$((count+1))
          done

      - name: Assert that the listener goes back up after the jobs are done
        shell: bash
        run: |
          count=0
          while true; do
            LISTENER_COUNT="$(kubectl get pods -l actions.github.com/scale-set-name=${{ steps.install_arc.outputs.ARC_NAME }} -n arc-systems --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
            RUNNERS_COUNT="$(kubectl get pods -l app.kubernetes.io/component=runner -n arc-runners --field-selector=status.phase=Running -o=jsonpath='{.items}' | jq 'length')"
            RESOURCES="$(kubectl get pods -A)"

            if [ "$LISTENER_COUNT" -eq 1 ]; then
              echo "Listener is up!"
              echo "$RESOURCES"
              exit 0
            fi
            if [ "$count" -ge 120 ]; then
              echo "Timeout waiting for listener to be recreated"
              echo "$RESOURCES"
              exit 1
            fi

            echo "Waiting for listener to be recreated"
            echo "Listener count: $LISTENER_COUNT target: 1 | Runners count: $RUNNERS_COUNT target: 0"

            sleep 1
            count=$((count+1))
          done

      - name: Gather logs and cleanup
        shell: bash
        if: always()
        run: |
          helm uninstall "${{ steps.install_arc.outputs.ARC_NAME }}" --namespace "arc-runners" --debug
          kubectl wait --timeout=10s --for=delete AutoScalingRunnerSet -n "${{ steps.install_arc.outputs.ARC_NAME }}" -l app.kubernetes.io/instance="${{ steps.install_arc.outputs.ARC_NAME }}"
          kubectl logs deployment/arc-gha-rs-controller -n "arc-systems"

  init-with-min-runners:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id
    env:
      WORKFLOW_FILE: arc-test-workflow.yaml
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.head_ref }}

      - uses: ./.github/actions/setup-arc-e2e
        id: setup
        with:
          app-id: ${{secrets.E2E_TESTS_ACCESS_APP_ID}}
          app-pk: ${{secrets.E2E_TESTS_ACCESS_PK}}
          image-name: ${{env.IMAGE_NAME}}
          image-tag: ${{env.IMAGE_VERSION}}
          target-org: ${{env.TARGET_ORG}}

      - name: Install gha-runner-scale-set-controller
        id: install_arc_controller
        run: |
          helm install arc \
          --namespace "arc-systems" \
          --create-namespace \
          --set image.repository=${{ env.IMAGE_NAME }} \
          --set image.tag=${{ env.IMAGE_VERSION }} \
          --set flags.updateStrategy="eventual" \
          ./charts/gha-runner-scale-set-controller \
          --debug
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l app.kubernetes.io/name=gha-rs-controller -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for controller pod with label app.kubernetes.io/name=gha-rs-controller"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l app.kubernetes.io/name=gha-rs-controller
          kubectl get pod -n arc-systems
          kubectl describe deployment arc-gha-rs-controller -n arc-systems

      - name: Install gha-runner-scale-set
        id: install_arc
        run: |
          ARC_NAME=${{github.job}}-$(date +'%M%S')$((($RANDOM + 100) % 100 + 1))
          helm install "$ARC_NAME" \
          --namespace "arc-runners" \
          --create-namespace \
          --set githubConfigUrl="https://github.com/${{ env.TARGET_ORG }}/${{env.TARGET_REPO}}" \
          --set githubConfigSecret.github_token="${{ steps.setup.outputs.token }}" \
          --set minRunners=5 \
          ./charts/gha-runner-scale-set \
          --debug
          echo "ARC_NAME=$ARC_NAME" >> $GITHUB_OUTPUT
          count=0
          while true; do
            POD_NAME=$(kubectl get pods -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME -o name)
            if [ -n "$POD_NAME" ]; then
              echo "Pod found: $POD_NAME"
              break
            fi
            if [ "$count" -ge 60 ]; then
              echo "Timeout waiting for listener pod with label actions.github.com/scale-set-name=$ARC_NAME"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done
          kubectl wait --timeout=30s --for=condition=ready pod -n arc-systems -l actions.github.com/scale-set-name=$ARC_NAME
          kubectl get pod -n arc-systems
      - name: Ensure 5 runners are up
        run: |
          count=0
          while true; do
            pod_count=$(kubectl get pods -n arc-runners --no-headers | wc -l)
            if [[ "$pod_count" = 5 ]]; then
              echo "5 pods are up!"
              break
            fi
            if [[ "$count" -ge 12 ]]; then
              echo "Timeout waiting for 5 pods to be created"
              exit 1
            fi
            sleep 1
            count=$((count+1))
          done