From 69af95a0525541d54677f645237841db636d8a3f Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 5 Jun 2026 10:20:04 -0700 Subject: [PATCH 1/4] Don't fail scheduled outerloop builds on Helix work item failures The libraries outerloop pipeline runs on a daily schedule with always:false, meaning AzDO only re-queues a commit if there were changes since the last successful scheduled run. Because flaky outerloop tests cause the 'Send to Helix' task to fail on essentially every scheduled run, the build never succeeds, so AzDO re-queues the same commit every day and submits ever more Helix work for an unchanged sha. Set shouldContinueOnError on the Send to Helix step for scheduled builds only (Build.Reason == 'Schedule'), so Helix work item failures no longer fail the build. Compile/build breaks still fail the build, and PR/CI/manual runs are unaffected. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/libraries/outerloop.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index 14d26e86dd0b2f..14ef4258e0d6a9 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -23,6 +23,10 @@ extends: parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + # Don't fail scheduled builds on Helix work item failures. Otherwise a perpetually + # failing scheduled build (flaky outerloop tests) causes AzDO to re-queue the same + # commit every day, wasting Helix resources. See always:false on the schedule above. + shouldContinueOnError: ${{ eq(variables['Build.Reason'], 'Schedule') }} buildConfig: Release platforms: - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: @@ -58,6 +62,8 @@ extends: parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + # Don't fail scheduled builds on Helix work item failures (see comment above). + shouldContinueOnError: ${{ eq(variables['Build.Reason'], 'Schedule') }} buildConfig: Debug platforms: - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: @@ -87,6 +93,8 @@ extends: parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + # Don't fail scheduled builds on Helix work item failures (see comment above). + shouldContinueOnError: ${{ eq(variables['Build.Reason'], 'Schedule') }} buildConfig: Release platforms: - windows_x86 From ab6a89e92fe8ad89204bea114e5a7119b5337ed8 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 5 Jun 2026 10:54:21 -0700 Subject: [PATCH 2/4] Make scheduled outerloop Helix step succeed instead of continueOnError continueOnError only marks the build partiallySucceeded, which AzDO's always:false scheduler still treats as not-successful, so the same commit keeps getting re-queued daily. Instead, for scheduled builds, tell the Helix SDK not to fail the build on work item / test failures by passing FailOnWorkItemFailure=false and FailOnTestFailure=false. The Send to Helix step then fully succeeds, so a perpetually-flaky scheduled run no longer causes AzDO to re-queue the same sha. - helix.yml: add failOnTestFailures parameter (default true = current behavior) wired to the FailOnWorkItemFailure/FailOnTestFailure Helix SDK properties. - outerloop.yml: pass failOnTestFailures=false only for scheduled builds (Build.Reason == 'Schedule'); replaces the earlier shouldContinueOnError approach. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/libraries/helix.yml | 7 +++++++ eng/pipelines/libraries/outerloop.yml | 19 +++++++++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml index f63792367fe78b..269a13055a0c36 100644 --- a/eng/pipelines/libraries/helix.yml +++ b/eng/pipelines/libraries/helix.yml @@ -13,6 +13,11 @@ parameters: extraHelixArguments: '' shouldContinueOnError: false scenarios: '' + # When false, Helix work item and test failures do not fail the build, so the "Send to Helix" + # step still succeeds. Unlike shouldContinueOnError (which only marks the build as + # partiallySucceeded), this produces a fully successful build. Scheduled builds with + # always:false set this so that flaky tests don't cause AzDO to re-queue the same commit daily. + failOnTestFailures: true steps: - script: $(_msbuildCommand) $(_warnAsErrorParamHelixOverride) -restore @@ -26,6 +31,8 @@ steps: /p:TestScope=${{ parameters.testScope }} /p:TestRunNamePrefixSuffix=${{ parameters.testRunNamePrefixSuffix }} /p:HelixBuild=$(Build.BuildNumber) + /p:FailOnWorkItemFailure=${{ parameters.failOnTestFailures }} + /p:FailOnTestFailure=${{ parameters.failOnTestFailures }} ${{ parameters.extraHelixArguments }} /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: Send to Helix diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index 14ef4258e0d6a9..b710d294539155 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -23,10 +23,6 @@ extends: parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - # Don't fail scheduled builds on Helix work item failures. Otherwise a perpetually - # failing scheduled build (flaky outerloop tests) causes AzDO to re-queue the same - # commit every day, wasting Helix resources. See always:false on the schedule above. - shouldContinueOnError: ${{ eq(variables['Build.Reason'], 'Schedule') }} buildConfig: Release platforms: - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: @@ -56,14 +52,17 @@ extends: testScope: outerloop creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + # On scheduled runs (always:false) don't fail the build on Helix work item or + # test failures, so flaky outerloop tests don't keep AzDO re-queueing the same + # commit. The Send to Helix step fully succeeds (not partiallySucceeded). + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + failOnTestFailures: false - ${{ if eq(variables['isRollingBuild'], false) }}: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - # Don't fail scheduled builds on Helix work item failures (see comment above). - shouldContinueOnError: ${{ eq(variables['Build.Reason'], 'Schedule') }} buildConfig: Debug platforms: - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: @@ -87,14 +86,15 @@ extends: testScope: outerloop creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + # Don't fail scheduled builds on Helix work item/test failures (see above). + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + failOnTestFailures: false - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - # Don't fail scheduled builds on Helix work item failures (see comment above). - shouldContinueOnError: ${{ eq(variables['Build.Reason'], 'Schedule') }} buildConfig: Release platforms: - windows_x86 @@ -114,3 +114,6 @@ extends: testScope: outerloop creator: dotnet-bot extraHelixArguments: /p:BuildTargetFramework=net48 + # Don't fail scheduled builds on Helix work item/test failures (see above). + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + failOnTestFailures: false From b36307ce98a890cf73a435450a7c0c086db0bba5 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 5 Jun 2026 11:15:57 -0700 Subject: [PATCH 3/4] TEMP: also apply failOnTestFailures on Manual runs (validation only, will revert) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/libraries/outerloop.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index b710d294539155..278c208903d753 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -55,7 +55,7 @@ extends: # On scheduled runs (always:false) don't fail the build on Helix work item or # test failures, so flaky outerloop tests don't keep AzDO re-queueing the same # commit. The Send to Helix step fully succeeds (not partiallySucceeded). - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + ${{ if in(variables['Build.Reason'], 'Schedule', 'Manual') }}: failOnTestFailures: false - ${{ if eq(variables['isRollingBuild'], false) }}: @@ -87,7 +87,7 @@ extends: creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) # Don't fail scheduled builds on Helix work item/test failures (see above). - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + ${{ if in(variables['Build.Reason'], 'Schedule', 'Manual') }}: failOnTestFailures: false - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: @@ -115,5 +115,5 @@ extends: creator: dotnet-bot extraHelixArguments: /p:BuildTargetFramework=net48 # Don't fail scheduled builds on Helix work item/test failures (see above). - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + ${{ if in(variables['Build.Reason'], 'Schedule', 'Manual') }}: failOnTestFailures: false From 522f1e21ce6fbea78f9bb58b4031ead6649133eb Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 5 Jun 2026 12:39:17 -0700 Subject: [PATCH 4/4] Revert temporary Manual-gate validation change Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/libraries/outerloop.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index 278c208903d753..b710d294539155 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -55,7 +55,7 @@ extends: # On scheduled runs (always:false) don't fail the build on Helix work item or # test failures, so flaky outerloop tests don't keep AzDO re-queueing the same # commit. The Send to Helix step fully succeeds (not partiallySucceeded). - ${{ if in(variables['Build.Reason'], 'Schedule', 'Manual') }}: + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: failOnTestFailures: false - ${{ if eq(variables['isRollingBuild'], false) }}: @@ -87,7 +87,7 @@ extends: creator: dotnet-bot testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) # Don't fail scheduled builds on Helix work item/test failures (see above). - ${{ if in(variables['Build.Reason'], 'Schedule', 'Manual') }}: + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: failOnTestFailures: false - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: @@ -115,5 +115,5 @@ extends: creator: dotnet-bot extraHelixArguments: /p:BuildTargetFramework=net48 # Don't fail scheduled builds on Helix work item/test failures (see above). - ${{ if in(variables['Build.Reason'], 'Schedule', 'Manual') }}: + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: failOnTestFailures: false