Merge branch 'main' into yucai/scatter

Author: yucai-intel

.github/workflows/_linux_build.yml

@@ -56,7 +56,7 @@ jobs:
         DisableScratchPages: ${{ inputs.driver == 'rolling' && '1' || '0' }}
     outputs:
       TORCH_COMMIT_ID: ${{ steps.build_version.outputs.TORCH_COMMIT_ID }}
-    timeout-minutes: 900
+    timeout-minutes: 300
     steps:
       - name: Setup based env
         run: |

.github/workflows/_linux_ut.yml

@@ -23,6 +23,11 @@ on:
         type: string
         default: ''
         description: UT scope. `op_regression,op_regression_dev1,op_transformers,op_extended,op_ut,torch_xpu` Delimiter is comma
+      disabled_tests:
+        required: false
+        type: string
+        default: ''
+        description: List disabled tests, such as disable_ut or disable_distributed
       python:
         required: false
         type: string
@@ -43,9 +48,9 @@ permissions: read-all
 
 jobs:
   ut_test:
-    runs-on: ${{ inputs.runner }} 
-    if: ${{ inputs.ut != 'xpu_distributed' }}
-    timeout-minutes: 900
+    runs-on: ${{ inputs.runner }}
+    if: ${{ inputs.ut != 'xpu_distributed' && !contains(inputs.disabled_tests, 'disable_ut') }}
+    timeout-minutes: 300
     env:
       GH_TOKEN: ${{ github.token }}
       NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
@@ -298,7 +303,39 @@ jobs:
           if [ -e "ut_failure_list.csv" ];then
               cp ut_failure_list.csv ${{ github.workspace }}/ut_log/ut_failure_list.csv
           fi
-      - name: UT Test Results Check
+          echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
+      - name: Upload Inductor XPU UT Log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}
+          path: ${{ github.workspace }}/ut_log
+      - name: Upload XPU UT Failure list
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}
+          path: ${{ github.workspace }}/ut_log/ut_failure_list.csv
+  
+  ut_test_results_check:
+    needs: ut_test
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      GH_TOKEN: ${{ github.token }}
+      ut_skip_issue: 1624
+    steps:
+      - name: Set the UT name
+        run: |
+          echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
+      - name: Checkout torch-xpu-ops
+        uses: actions/checkout@v4
+      - name: Download XPU UT Logs
+        uses: actions/download-artifact@v4
+        with:
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}
+          path: ${{ github.workspace }}/ut_log
+      - name: Check UT Results
         shell: bash
         run: |
           repo="${{ github.repository }}"
@@ -319,26 +356,28 @@ jobs:
             $contains_status
             cd ${{ github.workspace }}/ut_log/${ut_suite}
             gh --repo $repo issue view $ut_skip_issue --json body -q .body | sed '/^$/d' > Known_issue.log
+            gh api "repos/${{ github.repository }}/issues?labels=skipped" \
+            --jq '.[] | select(.pull_request == null) | "Issue #\(.number): \(.title)\n\(.body)\n"' \
+            > issues.log
+            awk '/Cases:/ {flag=1; next} /^\|\||^$/ {flag=0} flag' issues.log | grep -Eo 'test[^[:space:]]+( \|\| [^[:space:]]+)?' | sed 's/ *|| */ /g' | sort -u > issues_temp.log
+            awk '$2 == "op_ut" {print $1}' issues_temp.log > issues_op_ut.log
+            cat issues_temp.log | awk '{print $1}' >> Known_issue.log
+            awk -F'::' '{print $1}' issues_op_ut.log | sort -u | paste -sd ',' >> Known_issue.log
+
             cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./
             bash ut_result_check.sh ${ut_suite}
           done
       - name: Upload Inductor XPU UT Log
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}-checked
           path: ${{ github.workspace }}/ut_log
-      - name: Upload XPU UT Failure list
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: XPU-UT-Failure-List-${{ github.event.pull_request.number || github.sha }}-${{ env.UT_NAME }}
-          path: ${{ github.workspace }}/ut_log/ut_failure_list.csv
 
   distributed_ut_test:
-    runs-on: pvc_e2e
-    if: contains(inputs.ut, 'xpu_distributed')
-    timeout-minutes: 900
+    runs-on: pytorch-06
+    if: ${{ contains(inputs.ut, 'xpu_distributed') && !contains(inputs.disabled_tests, 'disable_distribute') }}
+    timeout-minutes: 60
     env:
       GH_TOKEN: ${{ github.token }}
       NEOReadDebugKeys: ${{ inputs.driver == 'rolling' && '1' || '0' }}
@@ -455,7 +494,32 @@ jobs:
           if [ -f ptrace_scope.bk ]; then
             sudo cp ptrace_scope.bk /proc/sys/kernel/yama/ptrace_scope
           fi
-      - name: UT Test Results Check
+      - name: Upload Inductor XPU UT Log
+        if: ${{ ! cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-xpu_distributed
+          path: ${{ github.workspace }}/ut_log
+
+  distributed_ut_test_results_check:
+    needs: distributed_ut_test
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      GH_TOKEN: ${{ github.token }}
+      ut_skip_issue: 1624
+    steps:
+      - name: Set the UT name
+        run: |
+          echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
+      - name: Checkout torch-xpu-ops
+        uses: actions/checkout@v4
+      - name: Download XPU UT Logs
+        uses: actions/download-artifact@v4
+        with:
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-xpu_distributed
+          path: ${{ github.workspace }}/ut_log
+      - name: Check UT Results
         shell: bash
         run: |
           repo="${{ github.repository }}"
@@ -472,11 +536,18 @@ jobs:
           echo "UT_NAME=$(echo ${{ inputs.ut }} |sed 's/,/-/g')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}"
           cd ${{ github.workspace }}/ut_log/xpu_distributed
           gh --repo $repo issue view $ut_skip_issue --json body -q .body | sed '/^$/d' > Known_issue.log
+          gh api "repos/${{ github.repository }}/issues?labels=skipped" \
+          --jq '.[] | select(.pull_request == null) | "Issue #\(.number): \(.title)\n\(.body)\n"' \
+          > issues.log
+          awk '/Cases:/ {flag=1; next} /^\|\||^$/ {flag=0} flag' issues.log | grep -Eo 'test[^[:space:]]+( \|\| [^[:space:]]+)?' | sed 's/ *|| */ /g' | sort -u > issues_temp.log
+          awk '$2 == "op_ut" {print $1}' issues_temp.log > issues_op_ut.log
+          cat issues_temp.log | awk '{print $1}' >> Known_issue.log
+          awk -F'::' '{print $1}' issues_op_ut.log | sort -u | paste -sd ',' >> Known_issue.log
           cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./
           bash ut_result_check.sh 'xpu_distributed'
       - name: Upload Inductor XPU UT Log
-        if: ${{ ! cancelled() }}
+        if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-xpu_distributed
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-xpu_distributed-checked
           path: ${{ github.workspace }}/ut_log

.github/workflows/_windows_ut.yml

@@ -33,12 +33,12 @@ on:
         type: string
         default: '2025.1'
         description: Python version
-      files-changed:
+      src_changed:
         required: true
         type: string
         default: 'false'
         description: Check if filtered file changed
-      has-label:
+      has_label:
         required: true
         type: string
         default: 'false'
@@ -50,9 +50,9 @@ env:
     USE_XPU: 1
 
 jobs:
-  Torch-XPU-Windows-UT:
+  ut_test:
     runs-on: ${{ inputs.runner }}
-    if: ${{ inputs.files-changed == 'true' || inputs.has-label == 'true' }}
+    if: ${{ inputs.src_changed == 'true' || inputs.has_label == 'true' }}
     timeout-minutes: 900
     steps:
       - name: Checkout torch-xpu-ops

.github/workflows/nightly_ondemand.yml

@@ -67,8 +67,8 @@ concurrency:
 
 jobs:
   Linux-Nightly-Ondemand-Build:
+    if: ${{ github.repository_owner == 'intel' }}
     secrets: inherit
-    if: ${{ ! cancelled() }}
     name: linux-nightly-ondemand
     permissions:
       issues: write
@@ -96,8 +96,6 @@ jobs:
   Linux-Nightly-Ondemand-E2E-Tests:
     runs-on: pvc_e2e
     name: linux-nightly-ondemand / e2e_test
-    # Don't run on forked repos
-    if: ${{ github.repository_owner == 'intel' }}
     needs: Linux-Nightly-Ondemand-Build
     timeout-minutes: 3600
     env:

.github/workflows/nightly_ondemand_rolling.yml

@@ -67,9 +67,8 @@ concurrency:
 
 jobs:
   Linux-Nightly-Ondemand-Build-Rolling:
-    # Don't run on forked repos
-    secrets: inherit
     if: ${{ github.repository_owner == 'intel' }}
+    secrets: inherit
     name: linux-nightly-ondemand-rolling
     permissions:
       issues: write
@@ -110,8 +109,6 @@ jobs:
 
   Linux-Nightly-Ondemand-E2E-Tests-Rolling:
     runs-on: pvc_rolling
-    # Don't run on forked repos
-    if: ${{ github.repository_owner == 'intel' }}
     name: linux-nightly-ondemand-rolling / e2e_test
     needs: Linux-Nightly-Ondemand-Build-Rolling
     timeout-minutes: 3600

.github/workflows/nightly_ondemand_whl.yml

@@ -57,7 +57,7 @@ concurrency:
 
 jobs:
   Linux-Nightly-Ondemand-UT-WHL-Tests:
-    if: ${{ github.event_name == 'schedule' || inputs.ut != '' }}
+    if: ${{ (github.event_name == 'schedule' || inputs.ut != '') && github.repository_owner == 'intel' }}
     uses: ./.github/workflows/_linux_ut.yml
     with:
       ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_transformers,op_extended,op_ut' || inputs.ut }}
@@ -67,7 +67,6 @@ jobs:
 
   Linux-Nightly-Ondemand-E2E-WHL-Tests:
     runs-on: pvc_e2e
-    # Don't run on forked repos
     if: ${{ github.repository_owner == 'intel' }}
     timeout-minutes: 3600
     env: