Skip to content

Commit 450d6ef

Browse files
authored
Read E2E logs and fail the job if no tests matched the filter (#62222)
* Make sure e2e fail if no tests were run. * Add threshold mechanism.
1 parent efdcb95 commit 450d6ef

File tree

1 file changed

+29
-5
lines changed

1 file changed

+29
-5
lines changed

.azure/pipelines/components-e2e-tests.yml

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,35 @@ jobs:
5858
displayName: Build JS
5959
- script: .dotnet/dotnet build ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-restore
6060
displayName: Build
61-
- script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined!=true|Quarantined=false'
62-
-p:VsTestUseMSBuildOutput=false
63-
--logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx"
64-
--logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html"
65-
--results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined
61+
- script: |
62+
set -o pipefail
63+
64+
.dotnet/dotnet test ./src/Components/test/E2ETest \
65+
-c $(BuildConfiguration) \
66+
--no-build \
67+
--filter 'Quarantined!=true|Quarantined=false' \
68+
-p:VsTestUseMSBuildOutput=false \
69+
--logger:"trx%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.trx" \
70+
--logger:"html%3BLogFileName=Microsoft.AspNetCore.Components.E2ETests.html" \
71+
--results-directory $(Build.SourcesDirectory)/artifacts/TestResults/$(BuildConfiguration)/Unquarantined \
72+
| tee e2e-test-output.log
73+
74+
if grep -q "No test matches the given testcase filter" e2e-test-output.log
75+
then
76+
echo "##vso[task.logissue type=error] No tests matched the filter."
77+
78+
exit 1
79+
fi
80+
81+
# Check total tests run to detect abnormalities. In case the number of tests changes significantly, we should adjust the threshold.
82+
# Extract total from the summary line "Failed: xx, Passed: yy, Skipped: zz, Total: NNN, Duration: ..."
83+
total=$(sed -nE 's/.*Failed:[[:space:]]*[0-9]+,[[:space:]]*Passed:[[:space:]]*[0-9]+,[[:space:]]*Skipped:[[:space:]]*[0-9]+,[[:space:]]*Total:[[:space:]]*([0-9]+).*/\1/p' e2e-test-output.log)
84+
min_total=1000
85+
if [ -z "$total" ] || [ "$total" -lt "$min_total" ]
86+
then
87+
echo "##vso[task.logissue type=error] Insufficient total test count: $total. We expect at least $min_total tests to run."
88+
exit 1
89+
fi
6690
displayName: Run E2E tests
6791
- script: .dotnet/dotnet test ./src/Components/test/E2ETest -c $(BuildConfiguration) --no-build --filter 'Quarantined=true' -p:RunQuarantinedTests=true
6892
-p:VsTestUseMSBuildOutput=false

0 commit comments

Comments
 (0)