Update JIT stage timing heuristic

docs/articles/guides/console-args.md

@@ -313,6 +313,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5
 * `--maxWidth`                Max parameter column width, the default is 20.
 * `--envVars`                 Colon separated environment variables (key:value)
 * `--memoryRandomization`     Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.
+* `--jitTieringMode`          (Default: Auto) Controls the behavior of the JIT stage when tiering is enabled. Auto/Force/Skip.
 * `--wasmEngine`              (Default: v8) Specifies the executable (in PATH) or full path to a java script engine used to run the benchmarks, used by Wasm toolchain.
 * `--wasmArgs`                (Default: --expose_wasm) Arguments for the javascript engine used by Wasm toolchain.
 * `--customRuntimePack`       Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently.

src/BenchmarkDotNet/Attributes/Mutators/JitTieringAttribute.cs

@@ -0,0 +1,9 @@
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+
+namespace BenchmarkDotNet.Attributes;
+
+/// <inheritdoc cref="RunMode.JitTieringMode"/>
+public class JitTieringAttribute(JitTieringMode mode) : JobMutatorConfigBaseAttribute(Job.Default.WithJitTieringMode(mode))
+{
+}

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -198,6 +198,9 @@ public bool UseDisassemblyDiagnoser
         [Option("memoryRandomization", Required = false, HelpText = "Specifies whether Engine should allocate some random-sized memory between iterations. It makes [GlobalCleanup] and [GlobalSetup] methods to be executed after every iteration.")]
         public bool MemoryRandomization { get; set; }
 
+        [Option("jitTieringMode", Required = false, Default = JitTieringMode.Auto, HelpText = "Controls the behavior of the JIT stage when tiering is enabled. Auto/Force/Skip.")]
+        public JitTieringMode JitTieringMode { get; set; }
+
         [Option("wasmEngine", Required = false, HelpText = "Specifies the executable (in PATH) or full path to a java script engine used to run the benchmarks, used by Wasm toolchain.", Default = "v8")]
         public string? WasmJavaScriptEngine { get; set; } = "v8";
 

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -455,6 +455,8 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig? globalConfig)
                 baseJob = baseJob.RunOncePerIteration();
             if (options.MemoryRandomization)
                 baseJob = baseJob.WithMemoryRandomization();
+            if (options.JitTieringMode != Engines.JitTieringMode.Auto)
+                baseJob = baseJob.WithJitTieringMode(options.JitTieringMode);
             if (options.NoForcedGCs)
                 baseJob = baseJob.WithGcForce(false);
             if (options.EvaluateOverhead is bool evaluateOverhead)

src/BenchmarkDotNet/Engines/EngineJitStage.cs

@@ -11,9 +11,6 @@ namespace BenchmarkDotNet.Engines;
 // the following stages (Pilot and Warmup) will likely take it the rest of the way. Long-running benchmarks may never fully reach tier1.
 internal sealed class EngineJitStage : EngineStage
 {
-    // It is not worth spending a long time in jit stage for macro-benchmarks.
-    private static readonly TimeInterval MaxTieringTime = TimeInterval.FromSeconds(10);
-
     // Jit call counting delay is only for when the app starts up. We don't need to wait for every benchmark if multiple benchmarks are ran in-process.
     private static TimeSpan s_tieredDelay = JitInfo.TieredDelay;
 
@@ -77,8 +74,9 @@ private IEnumerator<IterationData> EnumerateIterations()
         }
         yield return GetWorkloadIterationData(userInvokeCount);
 
-        // If the jit is not tiered, we're done.
-        if (!JitInfo.IsTiered)
+        JitTieringMode jitTieringMode = parameters.TargetJob.ResolveValue(RunMode.JitTieringModeCharacteristic, parameters.Resolver);
+        // If the jit is not tiered, or the user wants to skip, we're done.
+        if (!JitInfo.IsTiered || jitTieringMode == JitTieringMode.Skip)
         {
             yield break;
         }
@@ -88,49 +86,42 @@ private IEnumerator<IterationData> EnumerateIterations()
         // Don't make the next jit stage wait if it's ran in the same process.
         s_tieredDelay = TimeSpan.Zero;
 
-        // Attempt to promote methods to tier1, but don't spend too much time in jit stage.
-        StartedClock startedClock = parameters.TargetJob.ResolveValue(InfrastructureMode.ClockCharacteristic, parameters.Resolver)!.Start();
+        // If the first iteration suggests a long-running benchmark (a single invocation already
+        // takes ~2/3 of IterationTime or more), run one confirmation iteration and bail out if
+        // it agrees. Same cutoff value that pilot stage uses.
+        // We do not bail out immediately if the first iteration is long-running because it could
+        // be due to cctors or other lazy initialization that won't be hit in steady-state. #2004
+        // JitTieringMode.Force opts out of this heuristic and always promotes through every tier.
+        TimeInterval iterationTime = parameters.TargetJob.ResolveValue(RunMode.IterationTimeCharacteristic, parameters.Resolver);
+        long remainingCalls = JitInfo.TieredCallCountThreshold;
+        if (jitTieringMode == JitTieringMode.Auto
+            && iterationTime.Nanoseconds / (lastMeasurement.Nanoseconds / (double)userInvokeCount) < 1.5)
+        {
+            ++iterationIndex;
+            yield return GetWorkloadIterationData(userInvokeCount);
+            if (iterationTime.Nanoseconds / (lastMeasurement.Nanoseconds / (double)userInvokeCount) < 1.5)
+            {
+                didStopEarly = true;
+                yield break;
+            }
+            remainingCalls -= userInvokeCount;
+        }
 
-        int remainingTiers = JitInfo.MaxTierPromotions;
-        long lastInvokeCount = userInvokeCount;
-        while (remainingTiers > 0)
+        // Promote methods to tier1.
+        for (int remainingTiers = JitInfo.MaxTierPromotions; remainingTiers > 0; --remainingTiers)
         {
-            --remainingTiers;
-            long remainingCalls = JitInfo.TieredCallCountThreshold;
             while (remainingCalls > 0)
             {
-                long invokeCount;
-                if (hasUserInvocationCount)
-                {
-                    invokeCount = userInvokeCount;
-                }
-                else
-                {
-                    // If we can run a batch of calls within the time limit (based on the last measurement), do that instead of multiple single-invocation iterations.
-                    // For long-running benchmarks where even a small batch wouldn't fit, fall back to one invocation per iteration.
-                    var remainingTimeLimit = MaxTieringTime.ToNanoseconds() - startedClock.GetElapsed().GetNanoseconds();
-                    var lastMeasurementSingleInvocationTime = lastMeasurement.Nanoseconds / lastInvokeCount;
-                    long allowedCallsWithinTimeLimit = (long)Math.Floor(remainingTimeLimit / lastMeasurementSingleInvocationTime);
-                    invokeCount = allowedCallsWithinTimeLimit > 0
-                        ? Math.Min(remainingCalls, allowedCallsWithinTimeLimit)
-                        : 1;
-                }
-                lastInvokeCount = invokeCount;
-
+                // Run the whole tier's call budget in a single iteration unless the user pinned InvocationCount.
+                long invokeCount = hasUserInvocationCount ? userInvokeCount : remainingCalls;
                 remainingCalls -= invokeCount;
                 ++iterationIndex;
                 // The generated __Overhead method is aggressively optimized, so we don't need to run it again.
                 yield return GetWorkloadIterationData(invokeCount);
-
-                if ((remainingTiers > 0 || remainingCalls > 0)
-                    && startedClock.GetElapsed().GetTimeValue() >= MaxTieringTime)
-                {
-                    didStopEarly = true;
-                    yield break;
-                }
             }
 
             Engine.SleepIfPositive(JitInfo.BackgroundCompilationDelay);
+            remainingCalls = JitInfo.TieredCallCountThreshold;
         }
 
         // Empirical evidence shows that the first call after the method is tiered up may take longer,

src/BenchmarkDotNet/Engines/EngineResolver.cs

@@ -33,6 +33,7 @@ private EngineResolver()
             Register(AccuracyMode.MinInvokeCountCharacteristic, () => 4);
             Register(AccuracyMode.EvaluateOverheadCharacteristic, () => false);
             Register(RunMode.MemoryRandomizationCharacteristic, () => false);
+            Register(RunMode.JitTieringModeCharacteristic, () => JitTieringMode.Auto);
             Register(AccuracyMode.OutlierModeCharacteristic, job =>
             {
                 // if Memory Randomization was enabled and the benchmark is truly multimodal

src/BenchmarkDotNet/Engines/JitTieringMode.cs

@@ -0,0 +1,25 @@
+namespace BenchmarkDotNet.Engines;
+
+/// <summary>
+/// Controls the behavior of the JIT stage when tiering is enabled.
+/// </summary>
+public enum JitTieringMode
+{
+    /// <summary>
+    /// Default.
+    /// If the benchmark is long-running, tiering is skipped and the benchmark method is left at the tier following the initial invoke (usually tier0).
+    /// Otherwise, the JIT stage attempts to force the benchmark method and its callees to be promoted to their final tier by calling it repeatedly before measurements begin.
+    /// </summary>
+    Auto,
+
+    /// <summary>
+    /// Forces the JIT stage to attempt to force the benchmark method and its callees to be promoted to their final tier by calling it repeatedly before measurements begin.
+    /// Useful when you want the most stable tier1 measurements even for longer-running benchmarks.
+    /// </summary>
+    Force,
+
+    /// <summary>
+    /// Forces the JIT stage to skip tier promotion entirely, leaving the benchmark method at the tier following the initial invoke (usually tier0).
+    /// </summary>
+    Skip
+}

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -199,6 +199,9 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) =
         /// </summary>
         public static Job WithMemoryRandomization(this Job job, bool enable = true) => job.WithCore(j => j.Run.MemoryRandomization = enable);
 
+        /// <inheritdoc cref="RunMode.JitTieringMode"/>
+        public static Job WithJitTieringMode(this Job job, JitTieringMode mode) => job.WithCore(j => j.Run.JitTieringMode = mode);
+
         // Infrastructure
         public static Job WithToolchain(this Job job, IToolchain toolchain) => job.WithCore(j => j.Infrastructure.Toolchain = toolchain);
 

src/BenchmarkDotNet/Jobs/RunMode.cs

@@ -25,6 +25,7 @@ public sealed class RunMode : JobMode<RunMode>
         public static readonly Characteristic<int> MinWarmupIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MinWarmupIterationCount));
         public static readonly Characteristic<int> MaxWarmupIterationCountCharacteristic = CreateCharacteristic<int>(nameof(MaxWarmupIterationCount));
         public static readonly Characteristic<bool> MemoryRandomizationCharacteristic = CreateCharacteristic<bool>(nameof(MemoryRandomization));
+        public static readonly Characteristic<JitTieringMode> JitTieringModeCharacteristic = CreateCharacteristic<JitTieringMode>(nameof(JitTieringMode));
 
         public static readonly RunMode Dry = new RunMode(nameof(Dry))
         {
@@ -190,6 +191,20 @@ public bool MemoryRandomization
             set => MemoryRandomizationCharacteristic[this] = value;
         }
 
+        /// <summary>
+        /// Controls the behavior of the JIT stage when tiering is enabled.
+        /// <list type="bullet">
+        /// <item><see cref="JitTieringMode.Auto"/> (default): Promote through every tier for short-running benchmarks only.</item>
+        /// <item><see cref="JitTieringMode.Force"/>: Always promote through every tier, regardless of invocation time.</item>
+        /// <item><see cref="JitTieringMode.Skip"/>: Skip tier promotion entirely.</item>
+        /// </list>
+        /// </summary>
+        public JitTieringMode JitTieringMode
+        {
+            get => JitTieringModeCharacteristic[this];
+            set => JitTieringModeCharacteristic[this] = value;
+        }
+
         internal BdnExecution ToPerfonar() => new()
         {
             LaunchCount = HasValue(LaunchCountCharacteristic) ? LaunchCount : null,