From af0862b33a5d24eb470a22a0441d3a88491038ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 31 Jul 2026 21:34:28 +0200 Subject: [PATCH 1/2] Add Run.Random to shuffle test execution order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Run.Random and Run.RandomSeed. When Run.Random is enabled the order of test files, and the blocks and tests inside them, is shuffled. Items are only reordered within their own level, so the Describes in a file, the Describes and Contexts in a Describe, and the Its in a block. The shuffle uses a seeded System.Random, so a run is repeatable. Set Run.RandomSeed to repeat a specific order. The default 0 picks a new seed each run and reports it at the start, so a failing order can be repeated. Fix #2425 🤖 --- src/Main.ps1 | 7 + src/Pester.Runtime.ps1 | 69 +++++ src/csharp/Pester/RunConfiguration.cs | 38 +++ src/en-US/about_PesterConfiguration.help.txt | 16 ++ src/functions/Output.ps1 | 6 + tst/Pester.RSpec.RandomOrder.ts.ps1 | 256 +++++++++++++++++++ 6 files changed, 392 insertions(+) create mode 100644 tst/Pester.RSpec.RandomOrder.ts.ps1 diff --git a/src/Main.ps1 b/src/Main.ps1 index cba6dad83..aa057f9dc 100644 --- a/src/Main.ps1 +++ b/src/Main.ps1 @@ -548,6 +548,13 @@ function Invoke-Pester { # Write-PesterDebugMessage is used regardless of WriteScreenPlugin. Resolve-OutputConfiguration -PesterPreference $PesterPreference + # Resolve the randomized-order seed once for the whole run (#2425), so it is reported a + # single time and shared by every container - including parallel workers, which each + # receive this resolved configuration. RandomSeed 0 means "pick a new seed for this run". + if ($PesterPreference.Run.Random.Value -and 0 -eq $PesterPreference.Run.RandomSeed.Value) { + $PesterPreference.Run.RandomSeed = [System.Random]::new().Next(1, [int]::MaxValue) + } + if ('None' -ne $PesterPreference.Output.Verbosity.Value) { $plugins.Add((Get-WriteScreenPlugin -Verbosity $PesterPreference.Output.Verbosity.Value)) } diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index eefce47bc..59dfca46b 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -80,6 +80,11 @@ function New-PesterState { Stack = [Collections.Stack]@() + # [System.Random] used to shuffle the execution order of containers, blocks and tests + # when Run.Random is enabled. Seeded from Run.RandomSeed so a run can be repeated. + # Stays $null when Run.Random is disabled. + RandomOrderRandom = $null + # Captured here so the <> template expansion (which runs in the user's session state) can # invoke it via "& $____Pester.FormatNicelyForTemplate" while the function itself stays bound # to the Pester module session state, where Format-Nicely2 is available (#2744). @@ -2034,6 +2039,25 @@ function Invoke-Test { $state.PluginData = $PluginData $state.Configuration = $Configuration + # Randomized execution order (#2425). The seed is normally resolved once in Invoke-Pester + # and written back to the configuration so it can be reported and repeated. When Invoke-Test + # is called directly (e.g. from tests) with Run.Random enabled but no seed, pick one here. + if ($PesterPreference.Run.Random.Value) { + $randomSeed = $PesterPreference.Run.RandomSeed.Value + if (0 -eq $randomSeed) { + $randomSeed = [System.Random]::new().Next(1, [int]::MaxValue) + $PesterPreference.Run.RandomSeed = $randomSeed + } + + $state.RandomOrderRandom = [System.Random]::new($randomSeed) + + # Shuffle the order the containers (test files / script blocks) run in. The blocks and + # tests inside each container are shuffled later, during discovery post-processing. + if (@($BlockContainer).Count -gt 1) { + $BlockContainer = Get-RandomizedOrder -Random $state.RandomOrderRandom -InputObject $BlockContainer + } + } + # # TODO: this it potentially unreliable, because suppressed errors are written to Error as well. And the errors are captured only from the caller state. So let's use it only as a useful indicator during migration and see how it works in production code. # # finding if there were any non-terminating errors during the run, user can clear the array, and the array has fixed size so we can't just try to detect if there is any difference by counts before and after. So I capture the last known error in that state and try to find it in the array after the run @@ -2153,6 +2177,29 @@ function Invoke-Test { $executedContainers } +function Get-RandomizedOrder { + # Fisher-Yates shuffle. Returns a new array with the items in a random but + # deterministic order for a given seeded [System.Random], so a run can be repeated. + param ( + [Parameter(Mandatory = $true)] + [System.Random] $Random, + $InputObject + ) + + $items = [object[]]@($InputObject) + for ($i = $items.Length - 1; $i -gt 0; $i--) { + $j = $Random.Next(0, $i + 1) + if ($i -ne $j) { + $tmp = $items[$i] + $items[$i] = $items[$j] + $items[$j] = $tmp + } + } + + # comma to return the array as a single object, preventing pipeline unrolling + , $items +} + function PostProcess-DiscoveredBlock { param ( [Parameter(Mandatory = $true)] @@ -2178,6 +2225,28 @@ function PostProcess-DiscoveredBlock { $b.Root = $RootBlock $b.BlockContainer = $BlockContainer + # Randomize the order of this block's direct children (its child blocks and tests, kept + # together in .Order) when Run.Random is enabled. This shuffles same-level items only: + # the Describes in a file, the Describes/Contexts in a Describe, and the Its in a block. + # We do it here, before First/Last are marked below, and rebuild .Blocks and .Tests to + # follow the shuffled .Order so the one-time setup/teardown boundaries match the real + # execution order. Uses the run's seeded RNG so the order is repeatable (#2425). + if ($null -ne $state.RandomOrderRandom -and $b.Order.Count -gt 1) { + $shuffledOrder = Get-RandomizedOrder -Random $state.RandomOrderRandom -InputObject $b.Order + $b.Order.Clear() + $b.Blocks.Clear() + $b.Tests.Clear() + foreach ($item in $shuffledOrder) { + $null = $b.Order.Add($item) + if ('Test' -eq $item.ItemType) { + $null = $b.Tests.Add($item) + } + else { + $null = $b.Blocks.Add($item) + } + } + } + $tests = $b.Tests if ($b.IsRoot) { diff --git a/src/csharp/Pester/RunConfiguration.cs b/src/csharp/Pester/RunConfiguration.cs index 8ce6bbfd1..3f44d3532 100644 --- a/src/csharp/Pester/RunConfiguration.cs +++ b/src/csharp/Pester/RunConfiguration.cs @@ -37,6 +37,8 @@ public class RunConfiguration : ConfigurationSection private StringOption _skipRemainingOnFailure; private BoolOption _failOnNullOrEmptyForEach; private StringOption _repoRoot; + private BoolOption _random; + private IntOption _randomSeed; public static RunConfiguration Default { get { return new RunConfiguration(); } } public static RunConfiguration ShallowClone(RunConfiguration configuration) @@ -62,6 +64,8 @@ public RunConfiguration(IDictionary configuration) : this() configuration.AssignObjectIfNotNull(nameof(SkipRemainingOnFailure), v => SkipRemainingOnFailure = v); configuration.AssignValueIfNotNull(nameof(FailOnNullOrEmptyForEach), v => FailOnNullOrEmptyForEach = v); configuration.AssignObjectIfNotNull(nameof(RepoRoot), v => RepoRoot = v); + configuration.AssignValueIfNotNull(nameof(Random), v => Random = v); + configuration.AssignValueIfNotNull(nameof(RandomSeed), v => RandomSeed = v); } } @@ -80,6 +84,8 @@ public RunConfiguration(IDictionary configuration) : this() ParallelThrottleLimit = new IntOption("EXPERIMENTAL: Maximum number of test files to run at the same time when Run.Parallel is enabled, passed through to 'ForEach-Object -Parallel -ThrottleLimit'. The default 0 uses all available processors ([Environment]::ProcessorCount). Set a lower number to cap how many runspaces run concurrently. Only used when Run.Parallel is enabled.", 0); SkipRemainingOnFailure = new StringOption("Skips remaining tests after failure for selected scope, options are None, Run, Container and Block.", "None"); FailOnNullOrEmptyForEach = new BoolOption("Fails discovery when -ForEach is provided $null or @() in a block or test. Can be overridden for a specific Describe/Context/It using -AllowNullOrEmptyForEach.", true); + Random = new BoolOption("Randomize the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.RandomSeed so a run can be repeated, and helps surface hidden dependencies between tests.", false); + RandomSeed = new IntOption("Seed used to randomize execution order when Run.Random is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.RandomSeed to that value.", 0); RepoRoot = new StringOption("EXPERIMENTAL: Root directory of the repository. Found by searching for the .git directory recursively. When not found, the current working directory is used. Before each test file is discovered and run - in both sequential and parallel runs - Pester dot-sources a 'Pester.BeforeContainer.ps1' from this directory if one is present, so helper modules or dot-sourced setup the parent session would normally provide are available to every container. This is especially useful in parallel runs where each worker starts from a clean runspace and re-runs it.", FindRepoRoot()); } @@ -307,6 +313,38 @@ public StringOption RepoRoot } } + public BoolOption Random + { + get { return _random; } + set + { + if (_random == null) + { + _random = value; + } + else + { + _random = new BoolOption(_random, value.Value); + } + } + } + + public IntOption RandomSeed + { + get { return _randomSeed; } + set + { + if (_randomSeed == null) + { + _randomSeed = value; + } + else + { + _randomSeed = new IntOption(_randomSeed, value.Value); + } + } + } + private static string FindRepoRoot() { var originalDir = Directory.GetCurrentDirectory(); diff --git a/src/en-US/about_PesterConfiguration.help.txt b/src/en-US/about_PesterConfiguration.help.txt index 3cceba8c1..87c169168 100644 --- a/src/en-US/about_PesterConfiguration.help.txt +++ b/src/en-US/about_PesterConfiguration.help.txt @@ -81,10 +81,26 @@ SECTIONS AND OPTIONS Type: bool Default value: $true + Random: Randomize the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.RandomSeed so a run can be repeated, and helps surface hidden dependencies between tests. + Type: bool + Default value: $false + + RandomSeed: Seed used to randomize execution order when Run.Random is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.RandomSeed to that value. + Type: int + Default value: 0 + RepoRoot: EXPERIMENTAL: Root directory of the repository. Found by searching for the .git directory recursively. When not found, the current working directory is used. Before each test file is discovered and run - in both sequential and parallel runs - Pester dot-sources a 'Pester.BeforeContainer.ps1' from this directory if one is present, so helper modules or dot-sourced setup the parent session would normally provide are available to every container. This is especially useful in parallel runs where each worker starts from a clean runspace and re-runs it. Type: string Default value: '' + Random: Randomize the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.RandomSeed so a run can be repeated, and helps surface hidden dependencies between tests. + Type: bool + Default value: $false + + RandomSeed: Seed used to randomize execution order when Run.Random is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.RandomSeed to that value. + Type: int + Default value: 0 + Filter: Tag: Tags of Describe, Context or It to be run. Use 'None' to run only tests that have no tags. Type: string[] diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index 8f7123643..e7bacd1be 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -1,6 +1,7 @@ $script:ReportStrings = DATA { @{ VersionMessage = "Pester v{0}" + RandomOrderMessage = "Randomizing execution order using seed {0}. Set 'Run.RandomSeed = {0}' to repeat this order." CoverageMessage = 'Covered {2:0.##}% / {5:0.##}%. {3:N0} analyzed {0} in {4:N0} {1}.' MissedSingular = 'Missed command:' @@ -528,6 +529,11 @@ function Get-WriteScreenPlugin ($Verbosity) { $parallelSuffix = if ($Context.Parallel) { ' in parallel' } else { '' } Write-PesterHostMessage -ForegroundColor $ReportTheme.Container "`nRunning tests from $(@($Context.BlockContainers).Length) files$parallelSuffix." } + + if ($PesterPreference.Run.Random.Value) { + # Report the resolved seed so a randomized run (#2425) can be repeated. + Write-PesterHostMessage -ForegroundColor $ReportTheme.Discovery ($ReportStrings.RandomOrderMessage -f $PesterPreference.Run.RandomSeed.Value) + } } $p.ContainerDiscoveryEnd = { diff --git a/tst/Pester.RSpec.RandomOrder.ts.ps1 b/tst/Pester.RSpec.RandomOrder.ts.ps1 new file mode 100644 index 000000000..417a2f5d8 --- /dev/null +++ b/tst/Pester.RSpec.RandomOrder.ts.ps1 @@ -0,0 +1,256 @@ +param ([switch] $PassThru, [switch] $NoBuild) + +Get-Module P, PTestHelpers, Pester, Axiom | Remove-Module + +Import-Module $PSScriptRoot\p.psm1 -DisableNameChecking +Import-Module $PSScriptRoot\axiom\Axiom.psm1 -DisableNameChecking + +if (-not $NoBuild) { & "$PSScriptRoot\..\build.ps1" } +Import-Module $PSScriptRoot\..\bin\Pester.psd1 + +$global:PesterPreference = @{ + Debug = @{ + ShowFullErrors = $true + } + Output = @{ + Verbosity = 'None' + } +} +$PSDefaultParameterValues = @{} + +# A container with a known structure. Every It records its path into $global:__order when it runs, +# so we can observe the real execution order at every level: +# - the Describes directly in the file (A, B, C), +# - the Context/Describe nested in a Describe (A\A-inner), +# - the Its in a block (a1..a2, c1..c3, ...). +$script:SampleBlock = { + Describe 'A' { + It 'a1' { $global:__order.Add('A.a1') } + It 'a2' { $global:__order.Add('A.a2') } + Context 'A-inner' { + It 'ai1' { $global:__order.Add('A.inner.ai1') } + It 'ai2' { $global:__order.Add('A.inner.ai2') } + } + } + Describe 'B' { + It 'b1' { $global:__order.Add('B.b1') } + It 'b2' { $global:__order.Add('B.b2') } + } + Describe 'C' { + It 'c1' { $global:__order.Add('C.c1') } + It 'c2' { $global:__order.Add('C.c2') } + It 'c3' { $global:__order.Add('C.c3') } + } +} + +function Get-ExecutionOrder { + param ( + [ScriptBlock] $ScriptBlock = $script:SampleBlock, + [switch] $Random, + [int] $Seed = 0 + ) + + $global:__order = [System.Collections.Generic.List[string]]::new() + $c = [PesterConfiguration]::Default + $c.Run.ScriptBlock = $ScriptBlock + $c.Run.Random = [bool]$Random + $c.Run.RandomSeed = $Seed + $c.Run.PassThru = $true + $c.Output.Verbosity = 'None' + $r = Invoke-Pester -Configuration $c + + [PSCustomObject]@{ + Order = $global:__order.ToArray() + OrderString = $global:__order -join ',' + ResolvedSeed = $r.Configuration.Run.RandomSeed.Value + Result = $r + } +} + +i -PassThru:$PassThru { + b "Run.Random configuration options" { + t "Run.Random exists and defaults to disabled" { + $c = [PesterConfiguration]::Default + $c.Run.Random.Value | Verify-False + } + + t "Run.RandomSeed exists and defaults to 0" { + $c = [PesterConfiguration]::Default + $c.Run.RandomSeed.Value | Verify-Equal 0 + } + + t "Run.Random can be enabled and Run.RandomSeed can be set" { + $c = [PesterConfiguration]::Default + $c.Run.Random = $true + $c.Run.RandomSeed = 123 + $c.Run.Random.Value | Verify-True + $c.Run.RandomSeed.Value | Verify-Equal 123 + } + + t "options can be set from a hashtable" { + $c = [PesterConfiguration]@{ Run = @{ Random = $true; RandomSeed = 99 } } + $c.Run.Random.Value | Verify-True + $c.Run.RandomSeed.Value | Verify-Equal 99 + } + } + + b "Default order (Run.Random disabled)" { + t "keeps the discovery (declaration) order" { + $r = Get-ExecutionOrder + $r.OrderString | Verify-Equal 'A.a1,A.a2,A.inner.ai1,A.inner.ai2,B.b1,B.b2,C.c1,C.c2,C.c3' + } + } + + b "Randomized order is repeatable" { + t "the same seed produces the same order across runs" { + $first = Get-ExecutionOrder -Random -Seed 42 + $second = Get-ExecutionOrder -Random -Seed 42 + $first.OrderString | Verify-Equal $second.OrderString + } + + t "a randomized run differs from the declaration order" { + $ordered = Get-ExecutionOrder + $shuffled = Get-ExecutionOrder -Random -Seed 42 + ($shuffled.OrderString -ne $ordered.OrderString) | Verify-True + } + + t "different seeds produce different orders" { + $a = Get-ExecutionOrder -Random -Seed 42 + $b = Get-ExecutionOrder -Random -Seed 7 + ($a.OrderString -ne $b.OrderString) | Verify-True + } + } + + b "Randomized order shuffles every level, dropping nothing" { + t "runs exactly the same set of tests, only reordered" { + $ordered = Get-ExecutionOrder + $shuffled = Get-ExecutionOrder -Random -Seed 42 + + $shuffled.Order.Count | Verify-Equal $ordered.Order.Count + $expected = $ordered.Order | Sort-Object + $actual = $shuffled.Order | Sort-Object + ($actual -join ',') | Verify-Equal ($expected -join ',') + } + + t "reorders top-level Describes in a file" { + # Reduce each entry to its top-level Describe and keep the order they first appear in. + $shuffled = Get-ExecutionOrder -Random -Seed 42 + $topLevel = @($shuffled.Order | ForEach-Object { ($_ -split '\.')[0] } | Select-Object -Unique) + (($topLevel -join ',') -ne 'A,B,C') | Verify-True + } + + t "reorders the Its inside a block" { + # Seeds are chosen so the C block's tests are not in declaration order. + $found = $false + foreach ($seed in 1..20) { + $shuffled = Get-ExecutionOrder -Random -Seed $seed + $cTests = @($shuffled.Order | Where-Object { $_ -like 'C.*' }) + if (($cTests -join ',') -ne 'C.c1,C.c2,C.c3') { $found = $true; break } + } + $found | Verify-True + } + + t "reorders the Its inside a nested Context" { + $found = $false + foreach ($seed in 1..20) { + $shuffled = Get-ExecutionOrder -Random -Seed $seed + $inner = @($shuffled.Order | Where-Object { $_ -like 'A.inner.*' }) + if (($inner -join ',') -ne 'A.inner.ai1,A.inner.ai2') { $found = $true; break } + } + $found | Verify-True + } + } + + b "Auto seed (Run.RandomSeed = 0)" { + t "resolves a non-zero seed and reports it on the result configuration" { + $r = Get-ExecutionOrder -Random -Seed 0 + ($r.ResolvedSeed -ne 0) | Verify-True + } + + t "does not mutate the caller's configuration object" { + $c = [PesterConfiguration]::Default + $c.Run.ScriptBlock = $script:SampleBlock + $c.Run.Random = $true + $c.Run.RandomSeed = 0 + $c.Output.Verbosity = 'None' + $global:__order = [System.Collections.Generic.List[string]]::new() + $null = Invoke-Pester -Configuration $c + # The run works on a merged copy, so the caller's seed stays 0 (a fresh seed each run). + $c.Run.RandomSeed.Value | Verify-Equal 0 + } + + t "the reported seed reproduces the same order" { + $auto = Get-ExecutionOrder -Random -Seed 0 + $repro = Get-ExecutionOrder -Random -Seed $auto.ResolvedSeed + $auto.OrderString | Verify-Equal $repro.OrderString + } + } + + b "Randomized order keeps setup and teardown correct" { + t "one-time and each setup/teardown still run the right number of times" { + # If shuffling broke the First/Last markers, one-time setup/teardown would fire at the + # wrong item. Count invocations to prove they stay correct under a shuffled order. + $global:__oneTime = 0 + $global:__each = 0 + $sb = { + Describe 'S' { + BeforeAll { $global:__oneTime++ } + AfterAll { $global:__oneTime++ } + BeforeEach { $global:__each++ } + AfterEach { $global:__each++ } + It 's1' { 1 | Should -Be 1 } + It 's2' { 1 | Should -Be 1 } + It 's3' { 1 | Should -Be 1 } + } + } + $c = [PesterConfiguration]::Default + $c.Run.ScriptBlock = $sb + $c.Run.Random = $true + $c.Run.RandomSeed = 42 + $c.Run.PassThru = $true + $c.Output.Verbosity = 'None' + $r = Invoke-Pester -Configuration $c + + $r.PassedCount | Verify-Equal 3 + $r.FailedCount | Verify-Equal 0 + # BeforeAll + AfterAll once each. + $global:__oneTime | Verify-Equal 2 + # BeforeEach + AfterEach for each of the 3 tests. + $global:__each | Verify-Equal 6 + } + } + + b "Randomized file order" { + t "shuffles the order test files run in, repeatably" { + $folder = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().Guid) + $null = New-Item -ItemType Directory -Path $folder -Force + foreach ($n in 'One', 'Two', 'Three', 'Four', 'Five') { + Set-Content -Path (Join-Path $folder "$n.Tests.ps1") -Value "Describe '$n' { It 'i' { `$global:__forder.Add('$n') } }" + } + try { + function Get-FileOrder ([int] $Seed) { + $global:__forder = [System.Collections.Generic.List[string]]::new() + $c = [PesterConfiguration]::Default + $c.Run.Path = $folder + $c.Run.Random = $true + $c.Run.RandomSeed = $Seed + $c.Output.Verbosity = 'None' + $null = Invoke-Pester -Configuration $c + $global:__forder -join ',' + } + + $ordered = ('One', 'Two', 'Three', 'Four', 'Five') -join ',' + $a = Get-FileOrder -Seed 12345 + $b = Get-FileOrder -Seed 12345 + + # same set of files ran + (($a -split ',' | Sort-Object) -join ',') | Verify-Equal (($ordered -split ',' | Sort-Object) -join ',') + # repeatable + $a | Verify-Equal $b + # actually reordered + ($a -ne $ordered) | Verify-True + } + finally { Remove-Item -Path $folder -Recurse -Force } + } + } +} From 030675992951c22a414a57ec086cbc4e17258082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 31 Jul 2026 22:10:59 +0200 Subject: [PATCH 2/2] Rename to Run.Shuffle and add #pester:no-shuffle opt-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the option to Run.Shuffle / Run.ShuffleSeed. Shuffle reads better than Random and matches the wording in the run banner. Add a file level opt-out. A file or script block with a #pester:no-shuffle comment keeps its blocks and tests in declaration order even when Run.Shuffle is on, parsed the same way as #pester:no-parallel. Useful for tests that are order coupled on purpose, like mock history that accumulates across sibling Its. The resolved seed is written back to the run configuration, so it is returned on the result (Configuration.Run.ShuffleSeed) even when it was auto picked, and the run can be repeated on the same commit. 🤖 --- src/Main.ps1 | 10 +- src/Pester.Runtime.ps1 | 89 ++++++++--- src/csharp/Pester/RunConfiguration.cs | 32 ++-- src/en-US/about_PesterConfiguration.help.txt | 12 +- src/functions/Output.ps1 | 6 +- ...der.ts.ps1 => Pester.RSpec.Shuffle.ts.ps1} | 145 +++++++++++++----- 6 files changed, 200 insertions(+), 94 deletions(-) rename tst/{Pester.RSpec.RandomOrder.ts.ps1 => Pester.RSpec.Shuffle.ts.ps1} (61%) diff --git a/src/Main.ps1 b/src/Main.ps1 index aa057f9dc..c723bea50 100644 --- a/src/Main.ps1 +++ b/src/Main.ps1 @@ -548,11 +548,11 @@ function Invoke-Pester { # Write-PesterDebugMessage is used regardless of WriteScreenPlugin. Resolve-OutputConfiguration -PesterPreference $PesterPreference - # Resolve the randomized-order seed once for the whole run (#2425), so it is reported a - # single time and shared by every container - including parallel workers, which each - # receive this resolved configuration. RandomSeed 0 means "pick a new seed for this run". - if ($PesterPreference.Run.Random.Value -and 0 -eq $PesterPreference.Run.RandomSeed.Value) { - $PesterPreference.Run.RandomSeed = [System.Random]::new().Next(1, [int]::MaxValue) + # Resolve the shuffle seed once for the whole run (#2425), so it is reported a single + # time and shared by every container - including parallel workers, which each receive + # this resolved configuration. ShuffleSeed 0 means "pick a new seed for this run". + if ($PesterPreference.Run.Shuffle.Value -and 0 -eq $PesterPreference.Run.ShuffleSeed.Value) { + $PesterPreference.Run.ShuffleSeed = [System.Random]::new().Next(1, [int]::MaxValue) } if ('None' -ne $PesterPreference.Output.Verbosity.Value) { diff --git a/src/Pester.Runtime.ps1 b/src/Pester.Runtime.ps1 index 59dfca46b..d0f8588a4 100644 --- a/src/Pester.Runtime.ps1 +++ b/src/Pester.Runtime.ps1 @@ -81,9 +81,13 @@ function New-PesterState { Stack = [Collections.Stack]@() # [System.Random] used to shuffle the execution order of containers, blocks and tests - # when Run.Random is enabled. Seeded from Run.RandomSeed so a run can be repeated. - # Stays $null when Run.Random is disabled. - RandomOrderRandom = $null + # when Run.Shuffle is enabled. Seeded from Run.ShuffleSeed so a run can be repeated. + # Stays $null when Run.Shuffle is disabled. + ShuffleRandom = $null + + # Set of block containers that opt out of shuffling via a '#pester:no-shuffle' comment. + # Their blocks and tests keep declaration order even when Run.Shuffle is enabled. + NoShuffleContainers = $null # Captured here so the <> template expansion (which runs in the user's session state) can # invoke it via "& $____Pester.FormatNicelyForTemplate" while the function itself stays bound @@ -2039,22 +2043,31 @@ function Invoke-Test { $state.PluginData = $PluginData $state.Configuration = $Configuration - # Randomized execution order (#2425). The seed is normally resolved once in Invoke-Pester + # Shuffled execution order (#2425). The seed is normally resolved once in Invoke-Pester # and written back to the configuration so it can be reported and repeated. When Invoke-Test - # is called directly (e.g. from tests) with Run.Random enabled but no seed, pick one here. - if ($PesterPreference.Run.Random.Value) { - $randomSeed = $PesterPreference.Run.RandomSeed.Value - if (0 -eq $randomSeed) { - $randomSeed = [System.Random]::new().Next(1, [int]::MaxValue) - $PesterPreference.Run.RandomSeed = $randomSeed + # is called directly (e.g. from tests) with Run.Shuffle enabled but no seed, pick one here. + if ($PesterPreference.Run.Shuffle.Value) { + $shuffleSeed = $PesterPreference.Run.ShuffleSeed.Value + if (0 -eq $shuffleSeed) { + $shuffleSeed = [System.Random]::new().Next(1, [int]::MaxValue) + $PesterPreference.Run.ShuffleSeed = $shuffleSeed } - $state.RandomOrderRandom = [System.Random]::new($randomSeed) + $state.ShuffleRandom = [System.Random]::new($shuffleSeed) + + # A file or script block can opt out with a '#pester:no-shuffle' comment. Collect those + # containers so their blocks and tests keep declaration order during discovery post-processing. + $state.NoShuffleContainers = [System.Collections.Generic.HashSet[object]]::new() + foreach ($container in $BlockContainer) { + if (Test-BlockContainerIsNoShuffle -Container $container) { + $null = $state.NoShuffleContainers.Add($container) + } + } # Shuffle the order the containers (test files / script blocks) run in. The blocks and # tests inside each container are shuffled later, during discovery post-processing. if (@($BlockContainer).Count -gt 1) { - $BlockContainer = Get-RandomizedOrder -Random $state.RandomOrderRandom -InputObject $BlockContainer + $BlockContainer = Get-ShuffledOrder -Random $state.ShuffleRandom -InputObject $BlockContainer } } @@ -2177,7 +2190,7 @@ function Invoke-Test { $executedContainers } -function Get-RandomizedOrder { +function Get-ShuffledOrder { # Fisher-Yates shuffle. Returns a new array with the items in a random but # deterministic order for a given seeded [System.Random], so a run can be repeated. param ( @@ -2200,6 +2213,44 @@ function Get-RandomizedOrder { , $items } +function Test-BlockContainerIsNoShuffle { + # Returns $true when a container opts out of shuffling (#2425) via a file-level comment directive, + # parsed similarly to PowerShell's #requires: + # + # #pester:no-shuffle + # + # The marker is matched against real comment tokens using the PowerShell tokenizer, so it is + # recognized only inside comments and never inside strings or here-strings. It may appear anywhere + # in the file or script block. Blocks and tests in a marked container keep their declaration order. + [OutputType([bool])] + param ( + [Parameter(Mandatory = $true)] + $Container + ) + + $tokens = $null + $parseErrors = $null + + if ('File' -eq $Container.Type) { + $null = [System.Management.Automation.Language.Parser]::ParseFile($Container.Item.FullName, [ref] $tokens, [ref] $parseErrors) + } + elseif ('ScriptBlock' -eq $Container.Type) { + $null = [System.Management.Automation.Language.Parser]::ParseInput($Container.Item.ToString(), [ref] $tokens, [ref] $parseErrors) + } + else { + return $false + } + + foreach ($token in $tokens) { + if ($token.Kind -eq [System.Management.Automation.Language.TokenKind]::Comment -and + $token.Text -match '^#\s*pester:no-shuffle\b') { + return $true + } + } + + return $false +} + function PostProcess-DiscoveredBlock { param ( [Parameter(Mandatory = $true)] @@ -2225,14 +2276,16 @@ function PostProcess-DiscoveredBlock { $b.Root = $RootBlock $b.BlockContainer = $BlockContainer - # Randomize the order of this block's direct children (its child blocks and tests, kept - # together in .Order) when Run.Random is enabled. This shuffles same-level items only: + # Shuffle the order of this block's direct children (its child blocks and tests, kept + # together in .Order) when Run.Shuffle is enabled. This shuffles same-level items only: # the Describes in a file, the Describes/Contexts in a Describe, and the Its in a block. # We do it here, before First/Last are marked below, and rebuild .Blocks and .Tests to # follow the shuffled .Order so the one-time setup/teardown boundaries match the real - # execution order. Uses the run's seeded RNG so the order is repeatable (#2425). - if ($null -ne $state.RandomOrderRandom -and $b.Order.Count -gt 1) { - $shuffledOrder = Get-RandomizedOrder -Random $state.RandomOrderRandom -InputObject $b.Order + # execution order. Uses the run's seeded RNG so the order is repeatable (#2425). Containers + # that opt out with '#pester:no-shuffle' are skipped and keep their declaration order. + $containerOptedOut = $null -ne $state.NoShuffleContainers -and $state.NoShuffleContainers.Contains($BlockContainer) + if ($null -ne $state.ShuffleRandom -and -not $containerOptedOut -and $b.Order.Count -gt 1) { + $shuffledOrder = Get-ShuffledOrder -Random $state.ShuffleRandom -InputObject $b.Order $b.Order.Clear() $b.Blocks.Clear() $b.Tests.Clear() diff --git a/src/csharp/Pester/RunConfiguration.cs b/src/csharp/Pester/RunConfiguration.cs index 3f44d3532..46ce42a1c 100644 --- a/src/csharp/Pester/RunConfiguration.cs +++ b/src/csharp/Pester/RunConfiguration.cs @@ -37,8 +37,8 @@ public class RunConfiguration : ConfigurationSection private StringOption _skipRemainingOnFailure; private BoolOption _failOnNullOrEmptyForEach; private StringOption _repoRoot; - private BoolOption _random; - private IntOption _randomSeed; + private BoolOption _shuffle; + private IntOption _shuffleSeed; public static RunConfiguration Default { get { return new RunConfiguration(); } } public static RunConfiguration ShallowClone(RunConfiguration configuration) @@ -64,8 +64,8 @@ public RunConfiguration(IDictionary configuration) : this() configuration.AssignObjectIfNotNull(nameof(SkipRemainingOnFailure), v => SkipRemainingOnFailure = v); configuration.AssignValueIfNotNull(nameof(FailOnNullOrEmptyForEach), v => FailOnNullOrEmptyForEach = v); configuration.AssignObjectIfNotNull(nameof(RepoRoot), v => RepoRoot = v); - configuration.AssignValueIfNotNull(nameof(Random), v => Random = v); - configuration.AssignValueIfNotNull(nameof(RandomSeed), v => RandomSeed = v); + configuration.AssignValueIfNotNull(nameof(Shuffle), v => Shuffle = v); + configuration.AssignValueIfNotNull(nameof(ShuffleSeed), v => ShuffleSeed = v); } } @@ -84,8 +84,8 @@ public RunConfiguration(IDictionary configuration) : this() ParallelThrottleLimit = new IntOption("EXPERIMENTAL: Maximum number of test files to run at the same time when Run.Parallel is enabled, passed through to 'ForEach-Object -Parallel -ThrottleLimit'. The default 0 uses all available processors ([Environment]::ProcessorCount). Set a lower number to cap how many runspaces run concurrently. Only used when Run.Parallel is enabled.", 0); SkipRemainingOnFailure = new StringOption("Skips remaining tests after failure for selected scope, options are None, Run, Container and Block.", "None"); FailOnNullOrEmptyForEach = new BoolOption("Fails discovery when -ForEach is provided $null or @() in a block or test. Can be overridden for a specific Describe/Context/It using -AllowNullOrEmptyForEach.", true); - Random = new BoolOption("Randomize the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.RandomSeed so a run can be repeated, and helps surface hidden dependencies between tests.", false); - RandomSeed = new IntOption("Seed used to randomize execution order when Run.Random is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.RandomSeed to that value.", 0); + Shuffle = new BoolOption("Shuffle the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.ShuffleSeed so a run can be repeated, and helps surface hidden dependencies between tests. A single file can opt out with a '#pester:no-shuffle' comment.", false); + ShuffleSeed = new IntOption("Seed used to shuffle execution order when Run.Shuffle is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.ShuffleSeed to that value.", 0); RepoRoot = new StringOption("EXPERIMENTAL: Root directory of the repository. Found by searching for the .git directory recursively. When not found, the current working directory is used. Before each test file is discovered and run - in both sequential and parallel runs - Pester dot-sources a 'Pester.BeforeContainer.ps1' from this directory if one is present, so helper modules or dot-sourced setup the parent session would normally provide are available to every container. This is especially useful in parallel runs where each worker starts from a clean runspace and re-runs it.", FindRepoRoot()); } @@ -313,34 +313,34 @@ public StringOption RepoRoot } } - public BoolOption Random + public BoolOption Shuffle { - get { return _random; } + get { return _shuffle; } set { - if (_random == null) + if (_shuffle == null) { - _random = value; + _shuffle = value; } else { - _random = new BoolOption(_random, value.Value); + _shuffle = new BoolOption(_shuffle, value.Value); } } } - public IntOption RandomSeed + public IntOption ShuffleSeed { - get { return _randomSeed; } + get { return _shuffleSeed; } set { - if (_randomSeed == null) + if (_shuffleSeed == null) { - _randomSeed = value; + _shuffleSeed = value; } else { - _randomSeed = new IntOption(_randomSeed, value.Value); + _shuffleSeed = new IntOption(_shuffleSeed, value.Value); } } } diff --git a/src/en-US/about_PesterConfiguration.help.txt b/src/en-US/about_PesterConfiguration.help.txt index 87c169168..8522867e1 100644 --- a/src/en-US/about_PesterConfiguration.help.txt +++ b/src/en-US/about_PesterConfiguration.help.txt @@ -81,23 +81,15 @@ SECTIONS AND OPTIONS Type: bool Default value: $true - Random: Randomize the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.RandomSeed so a run can be repeated, and helps surface hidden dependencies between tests. - Type: bool - Default value: $false - - RandomSeed: Seed used to randomize execution order when Run.Random is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.RandomSeed to that value. - Type: int - Default value: 0 - RepoRoot: EXPERIMENTAL: Root directory of the repository. Found by searching for the .git directory recursively. When not found, the current working directory is used. Before each test file is discovered and run - in both sequential and parallel runs - Pester dot-sources a 'Pester.BeforeContainer.ps1' from this directory if one is present, so helper modules or dot-sourced setup the parent session would normally provide are available to every container. This is especially useful in parallel runs where each worker starts from a clean runspace and re-runs it. Type: string Default value: '' - Random: Randomize the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.RandomSeed so a run can be repeated, and helps surface hidden dependencies between tests. + Shuffle: Shuffle the order in which test files, and the blocks (Describe/Context) and tests (It) inside them, are executed. Items are only reordered within their own level. Uses Run.ShuffleSeed so a run can be repeated, and helps surface hidden dependencies between tests. A single file can opt out with a '#pester:no-shuffle' comment. Type: bool Default value: $false - RandomSeed: Seed used to randomize execution order when Run.Random is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.RandomSeed to that value. + ShuffleSeed: Seed used to shuffle execution order when Run.Shuffle is enabled. The default 0 picks a new seed for each run and reports it at the start, so the run can be repeated by setting Run.ShuffleSeed to that value. Type: int Default value: 0 diff --git a/src/functions/Output.ps1 b/src/functions/Output.ps1 index e7bacd1be..9b34f57cf 100644 --- a/src/functions/Output.ps1 +++ b/src/functions/Output.ps1 @@ -1,7 +1,7 @@ $script:ReportStrings = DATA { @{ VersionMessage = "Pester v{0}" - RandomOrderMessage = "Randomizing execution order using seed {0}. Set 'Run.RandomSeed = {0}' to repeat this order." + ShuffleMessage = "Shuffling execution order using seed {0}. Set 'Run.ShuffleSeed = {0}' to repeat this order." CoverageMessage = 'Covered {2:0.##}% / {5:0.##}%. {3:N0} analyzed {0} in {4:N0} {1}.' MissedSingular = 'Missed command:' @@ -530,9 +530,9 @@ function Get-WriteScreenPlugin ($Verbosity) { Write-PesterHostMessage -ForegroundColor $ReportTheme.Container "`nRunning tests from $(@($Context.BlockContainers).Length) files$parallelSuffix." } - if ($PesterPreference.Run.Random.Value) { + if ($PesterPreference.Run.Shuffle.Value) { # Report the resolved seed so a randomized run (#2425) can be repeated. - Write-PesterHostMessage -ForegroundColor $ReportTheme.Discovery ($ReportStrings.RandomOrderMessage -f $PesterPreference.Run.RandomSeed.Value) + Write-PesterHostMessage -ForegroundColor $ReportTheme.Discovery ($ReportStrings.ShuffleMessage -f $PesterPreference.Run.ShuffleSeed.Value) } } diff --git a/tst/Pester.RSpec.RandomOrder.ts.ps1 b/tst/Pester.RSpec.Shuffle.ts.ps1 similarity index 61% rename from tst/Pester.RSpec.RandomOrder.ts.ps1 rename to tst/Pester.RSpec.Shuffle.ts.ps1 index 417a2f5d8..176b98a83 100644 --- a/tst/Pester.RSpec.RandomOrder.ts.ps1 +++ b/tst/Pester.RSpec.Shuffle.ts.ps1 @@ -46,15 +46,15 @@ $script:SampleBlock = { function Get-ExecutionOrder { param ( [ScriptBlock] $ScriptBlock = $script:SampleBlock, - [switch] $Random, + [switch] $Shuffle, [int] $Seed = 0 ) $global:__order = [System.Collections.Generic.List[string]]::new() $c = [PesterConfiguration]::Default $c.Run.ScriptBlock = $ScriptBlock - $c.Run.Random = [bool]$Random - $c.Run.RandomSeed = $Seed + $c.Run.Shuffle = [bool]$Shuffle + $c.Run.ShuffleSeed = $Seed $c.Run.PassThru = $true $c.Output.Verbosity = 'None' $r = Invoke-Pester -Configuration $c @@ -62,69 +62,69 @@ function Get-ExecutionOrder { [PSCustomObject]@{ Order = $global:__order.ToArray() OrderString = $global:__order -join ',' - ResolvedSeed = $r.Configuration.Run.RandomSeed.Value + ResolvedSeed = $r.Configuration.Run.ShuffleSeed.Value Result = $r } } i -PassThru:$PassThru { - b "Run.Random configuration options" { - t "Run.Random exists and defaults to disabled" { + b "Run.Shuffle configuration options" { + t "Run.Shuffle exists and defaults to disabled" { $c = [PesterConfiguration]::Default - $c.Run.Random.Value | Verify-False + $c.Run.Shuffle.Value | Verify-False } - t "Run.RandomSeed exists and defaults to 0" { + t "Run.ShuffleSeed exists and defaults to 0" { $c = [PesterConfiguration]::Default - $c.Run.RandomSeed.Value | Verify-Equal 0 + $c.Run.ShuffleSeed.Value | Verify-Equal 0 } - t "Run.Random can be enabled and Run.RandomSeed can be set" { + t "Run.Shuffle can be enabled and Run.ShuffleSeed can be set" { $c = [PesterConfiguration]::Default - $c.Run.Random = $true - $c.Run.RandomSeed = 123 - $c.Run.Random.Value | Verify-True - $c.Run.RandomSeed.Value | Verify-Equal 123 + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 123 + $c.Run.Shuffle.Value | Verify-True + $c.Run.ShuffleSeed.Value | Verify-Equal 123 } t "options can be set from a hashtable" { - $c = [PesterConfiguration]@{ Run = @{ Random = $true; RandomSeed = 99 } } - $c.Run.Random.Value | Verify-True - $c.Run.RandomSeed.Value | Verify-Equal 99 + $c = [PesterConfiguration]@{ Run = @{ Shuffle = $true; ShuffleSeed = 99 } } + $c.Run.Shuffle.Value | Verify-True + $c.Run.ShuffleSeed.Value | Verify-Equal 99 } } - b "Default order (Run.Random disabled)" { + b "Default order (Run.Shuffle disabled)" { t "keeps the discovery (declaration) order" { $r = Get-ExecutionOrder $r.OrderString | Verify-Equal 'A.a1,A.a2,A.inner.ai1,A.inner.ai2,B.b1,B.b2,C.c1,C.c2,C.c3' } } - b "Randomized order is repeatable" { + b "Shuffled order is repeatable" { t "the same seed produces the same order across runs" { - $first = Get-ExecutionOrder -Random -Seed 42 - $second = Get-ExecutionOrder -Random -Seed 42 + $first = Get-ExecutionOrder -Shuffle -Seed 42 + $second = Get-ExecutionOrder -Shuffle -Seed 42 $first.OrderString | Verify-Equal $second.OrderString } t "a randomized run differs from the declaration order" { $ordered = Get-ExecutionOrder - $shuffled = Get-ExecutionOrder -Random -Seed 42 + $shuffled = Get-ExecutionOrder -Shuffle -Seed 42 ($shuffled.OrderString -ne $ordered.OrderString) | Verify-True } t "different seeds produce different orders" { - $a = Get-ExecutionOrder -Random -Seed 42 - $b = Get-ExecutionOrder -Random -Seed 7 + $a = Get-ExecutionOrder -Shuffle -Seed 42 + $b = Get-ExecutionOrder -Shuffle -Seed 7 ($a.OrderString -ne $b.OrderString) | Verify-True } } - b "Randomized order shuffles every level, dropping nothing" { + b "Shuffled order shuffles every level, dropping nothing" { t "runs exactly the same set of tests, only reordered" { $ordered = Get-ExecutionOrder - $shuffled = Get-ExecutionOrder -Random -Seed 42 + $shuffled = Get-ExecutionOrder -Shuffle -Seed 42 $shuffled.Order.Count | Verify-Equal $ordered.Order.Count $expected = $ordered.Order | Sort-Object @@ -134,7 +134,7 @@ i -PassThru:$PassThru { t "reorders top-level Describes in a file" { # Reduce each entry to its top-level Describe and keep the order they first appear in. - $shuffled = Get-ExecutionOrder -Random -Seed 42 + $shuffled = Get-ExecutionOrder -Shuffle -Seed 42 $topLevel = @($shuffled.Order | ForEach-Object { ($_ -split '\.')[0] } | Select-Object -Unique) (($topLevel -join ',') -ne 'A,B,C') | Verify-True } @@ -143,7 +143,7 @@ i -PassThru:$PassThru { # Seeds are chosen so the C block's tests are not in declaration order. $found = $false foreach ($seed in 1..20) { - $shuffled = Get-ExecutionOrder -Random -Seed $seed + $shuffled = Get-ExecutionOrder -Shuffle -Seed $seed $cTests = @($shuffled.Order | Where-Object { $_ -like 'C.*' }) if (($cTests -join ',') -ne 'C.c1,C.c2,C.c3') { $found = $true; break } } @@ -153,7 +153,7 @@ i -PassThru:$PassThru { t "reorders the Its inside a nested Context" { $found = $false foreach ($seed in 1..20) { - $shuffled = Get-ExecutionOrder -Random -Seed $seed + $shuffled = Get-ExecutionOrder -Shuffle -Seed $seed $inner = @($shuffled.Order | Where-Object { $_ -like 'A.inner.*' }) if (($inner -join ',') -ne 'A.inner.ai1,A.inner.ai2') { $found = $true; break } } @@ -161,32 +161,32 @@ i -PassThru:$PassThru { } } - b "Auto seed (Run.RandomSeed = 0)" { + b "Auto seed (Run.ShuffleSeed = 0)" { t "resolves a non-zero seed and reports it on the result configuration" { - $r = Get-ExecutionOrder -Random -Seed 0 + $r = Get-ExecutionOrder -Shuffle -Seed 0 ($r.ResolvedSeed -ne 0) | Verify-True } t "does not mutate the caller's configuration object" { $c = [PesterConfiguration]::Default $c.Run.ScriptBlock = $script:SampleBlock - $c.Run.Random = $true - $c.Run.RandomSeed = 0 + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 0 $c.Output.Verbosity = 'None' $global:__order = [System.Collections.Generic.List[string]]::new() $null = Invoke-Pester -Configuration $c # The run works on a merged copy, so the caller's seed stays 0 (a fresh seed each run). - $c.Run.RandomSeed.Value | Verify-Equal 0 + $c.Run.ShuffleSeed.Value | Verify-Equal 0 } t "the reported seed reproduces the same order" { - $auto = Get-ExecutionOrder -Random -Seed 0 - $repro = Get-ExecutionOrder -Random -Seed $auto.ResolvedSeed + $auto = Get-ExecutionOrder -Shuffle -Seed 0 + $repro = Get-ExecutionOrder -Shuffle -Seed $auto.ResolvedSeed $auto.OrderString | Verify-Equal $repro.OrderString } } - b "Randomized order keeps setup and teardown correct" { + b "Shuffled order keeps setup and teardown correct" { t "one-time and each setup/teardown still run the right number of times" { # If shuffling broke the First/Last markers, one-time setup/teardown would fire at the # wrong item. Count invocations to prove they stay correct under a shuffled order. @@ -205,8 +205,8 @@ i -PassThru:$PassThru { } $c = [PesterConfiguration]::Default $c.Run.ScriptBlock = $sb - $c.Run.Random = $true - $c.Run.RandomSeed = 42 + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 42 $c.Run.PassThru = $true $c.Output.Verbosity = 'None' $r = Invoke-Pester -Configuration $c @@ -220,7 +220,7 @@ i -PassThru:$PassThru { } } - b "Randomized file order" { + b "Shuffled file order" { t "shuffles the order test files run in, repeatably" { $folder = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().Guid) $null = New-Item -ItemType Directory -Path $folder -Force @@ -232,8 +232,8 @@ i -PassThru:$PassThru { $global:__forder = [System.Collections.Generic.List[string]]::new() $c = [PesterConfiguration]::Default $c.Run.Path = $folder - $c.Run.Random = $true - $c.Run.RandomSeed = $Seed + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = $Seed $c.Output.Verbosity = 'None' $null = Invoke-Pester -Configuration $c $global:__forder -join ',' @@ -253,4 +253,65 @@ i -PassThru:$PassThru { finally { Remove-Item -Path $folder -Recurse -Force } } } + + b "Opting out with #pester:no-shuffle" { + t "a script block with the directive keeps its declaration order while shuffle is on" { + $sb = { + # pester:no-shuffle + Describe 'A' { + It 'a1' { $global:__order.Add('A.a1') } + It 'a2' { $global:__order.Add('A.a2') } + Context 'inner' { + It 'ai1' { $global:__order.Add('A.inner.ai1') } + It 'ai2' { $global:__order.Add('A.inner.ai2') } + } + } + Describe 'B' { + It 'b1' { $global:__order.Add('B.b1') } + It 'b2' { $global:__order.Add('B.b2') } + } + } + $declared = 'A.a1,A.a2,A.inner.ai1,A.inner.ai2,B.b1,B.b2' + # Try several seeds; none should be able to reorder the opted-out container. + foreach ($seed in 1, 7, 42, 2024) { + $r = Get-ExecutionOrder -ScriptBlock $sb -Shuffle -Seed $seed + $r.OrderString | Verify-Equal $declared + } + } + + t "the directive keeps one file ordered while other files still shuffle" { + $folder = Join-Path ([IO.Path]::GetTempPath()) ([Guid]::NewGuid().Guid) + $null = New-Item -ItemType Directory -Path $folder -Force + # Ordered.Tests.ps1 opts out; its 4 Its must stay in order. The other files are only there + # to make sure shuffle is actually active in the run. + Set-Content -Path (Join-Path $folder 'Ordered.Tests.ps1') -Value @' +# pester:no-shuffle +Describe 'Ordered' { + It 'o1' { $global:__order2.Add('o1') } + It 'o2' { $global:__order2.Add('o2') } + It 'o3' { $global:__order2.Add('o3') } + It 'o4' { $global:__order2.Add('o4') } +} +'@ + foreach ($n in 'Free1', 'Free2', 'Free3') { + Set-Content -Path (Join-Path $folder "$n.Tests.ps1") -Value "Describe '$n' { It 'a' { 1 | Should -Be 1 }; It 'b' { 1 | Should -Be 1 } }" + } + try { + $global:__order2 = [System.Collections.Generic.List[string]]::new() + $c = [PesterConfiguration]::Default + $c.Run.Path = $folder + $c.Run.Shuffle = $true + $c.Run.ShuffleSeed = 42 + $c.Output.Verbosity = 'None' + $c.Run.PassThru = $true + $r = Invoke-Pester -Configuration $c + + # the opted-out file kept its declaration order + ($global:__order2 -join ',') | Verify-Equal 'o1,o2,o3,o4' + # and everything still ran + $r.FailedCount | Verify-Equal 0 + } + finally { Remove-Item -Path $folder -Recurse -Force } + } + } }