From 449e6eb0f3ca99dd8bcaad219ec08e3b68e53ece Mon Sep 17 00:00:00 2001 From: monkey1sai <26239865+monkey1sai@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:44:13 +0800 Subject: [PATCH 1/2] fix(deploy): make orphaned Kit visible to liveness and fail Phase 4c closed (#640) Liveness was tracked as the LAUNCHER's pid, but the process holding the streaming ports, the GPU context and the Omniverse user directory is its Kit child. When the launcher died and the child survived, Remove-StalePidFile correctly deleted the pid file, Test-AlreadyRunning correctly reported "not running", and Phase 4c started a second Kit into the live one - which deadlocked in early startup with two futex-waiting threads, no listener and not one line of its own log, and was only noticed 480s later. Two halves: - Start-HostNativeService now records the TCP ports a launch claims in a .ports sidecar, written BEFORE the launch and deliberately left in place by Remove-StalePidFile. That record is the surviving trace of an orphaned child; only a stop that actually terminated something clears it. Start-HostNativeKit declares its signal + spectator signal ports. - Get-HostNativeOrphanListener answers "is a process outside our recorded tree still holding these ports?" over the union of the recorded claim and this run's expected ports, and Phase 4c consults it immediately before launching. An unaccounted holder is a hard stop naming the ports, the pids and scripts/stop-all.ps1 - the manual recovery that made the failing deployment pass unchanged - instead of a second instance. A holder that vanishes within a bounded settle window is teardown, not an orphan. Phase 1 already printed this exact observation ("occupied by our PID ... already running, will skip start"); it was never a gate. Now it is. Fixes #640. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GSyeoncjEm8rcfRDgnS6CB --- scripts/deploy.ps1 | 27 ++ scripts/lib/host-native-launcher.ps1 | 195 ++++++++++++++ .../tests/test-deploy-governance-static.ps1 | 19 ++ scripts/tests/test-host-native-launcher.ps1 | 248 ++++++++++++++++++ 4 files changed, 489 insertions(+) diff --git a/scripts/deploy.ps1 b/scripts/deploy.ps1 index 03ae019a6..77f91775e 100644 --- a/scripts/deploy.ps1 +++ b/scripts/deploy.ps1 @@ -1729,6 +1729,33 @@ if ($SkipKit) { if ($kitAlreadyRunning) { Write-DeployTag -Tag 'skip' -Message 'Phase 4c host-native Kit already running with matching runtime parameters' -LogPath $LogPath | Out-Null } else { + # Orphan gate (#640). Liveness above is the LAUNCHER's pid, but the Kit + # child holds the ports, the GPU context and the Omniverse user + # directory. When the launcher dies and the child survives, + # Remove-StalePidFile correctly drops the pid file, Test-AlreadyRunning + # correctly reports "not running", and starting anyway put a second Kit + # on top of a live one: the new process deadlocked in early startup with + # two futex-waiting threads, no listener and not one line of its own log, + # and the deploy only found out 480s later. + # + # Phase 1 already prints this exact observation ("occupied by our PID + # ... already running, will skip start"). It was never a gate; this is. + # Refusing is the only honest option here: this run cannot tell a + # deadlocked orphan from a healthy instance, and it must not adopt one or + # race one. scripts/stop-all.ps1 stops by port as well as by pid file, so + # it reaches an orphan whose pid file is already gone - which is exactly + # the manual recovery that made the failing deployment pass unchanged. + $kitOrphan = Get-HostNativeOrphanListener ` + -Name 'bim-streaming-server' ` + -RunDir $RunDir ` + -ExpectedPorts (@($resolvedKitSignalPort) + @($resolvedSpectatorSignalPorts)) + if ($null -ne $kitOrphan) { + $orphanPortList = @($kitOrphan.Ports) -join ', ' + $orphanPidList = @($kitOrphan.ProcessIds | ForEach-Object { if ([int]$_ -le 0) { 'owner-not-visible' } else { "$_" } }) -join ', ' + Write-DeployTag -Tag 'fail' -Message "stage=4c Phase 4c refusing to start a second Kit: TCP port(s) $orphanPortList still LISTEN under PID(s) $orphanPidList, which no live bim-streaming-server PID file accounts for (orphaned Kit). Stop it first with scripts/stop-all.ps1, then re-run this deploy" -LogPath $LogPath | Out-Null + Print-FinalSummary -ExitCode 4 -FailedPhase 'Phase 4c (orphaned Kit holds the streaming ports)' + exit 4 + } Write-DeployTag -Tag 'ok' -Message 'Phase 4c starting host-native Kit streaming' -LogPath $LogPath | Out-Null $startInfo = Start-HostNativeKit ` -RepoRoot $RepoRoot ` diff --git a/scripts/lib/host-native-launcher.ps1 b/scripts/lib/host-native-launcher.ps1 index d98006df4..fadf6b57e 100644 --- a/scripts/lib/host-native-launcher.ps1 +++ b/scripts/lib/host-native-launcher.ps1 @@ -156,7 +156,167 @@ function Test-AlreadyRunning { return ($null -ne (& $GetProcessFn $procId)) } +function Get-HostNativeServiceListenPorts { + # The TCP ports the LAST launch of this service declared it would own, read + # back from the .ports sidecar. Empty when no launch ever declared any. + # + # This sidecar exists because the pid file cannot answer the question that + # matters (#640). Liveness is recorded as the LAUNCHER's pid, but the thing + # holding the ports, the GPU context and the Omniverse user directory is its + # CHILD - for the Kit service, a `kit` process started by the pwsh wrapper. + # When the wrapper dies and the child is orphaned, Remove-StalePidFile + # correctly deletes the pid file (the recorded pid really is gone) and the + # only record that ANY instance is still holding those resources disappears + # with it. The sidecar deliberately outlives the pid file so the surviving + # child stays visible; only a deliberate stop removes it. + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)][string] $Name, + [Parameter(Mandatory = $true)][string] $RunDir + ) + $portFile = Join-Path $RunDir "$Name.ports" + if (-not (Test-Path -LiteralPath $portFile -PathType Leaf)) { return @() } + $ports = @() + foreach ($line in @(Get-Content -LiteralPath $portFile -ErrorAction SilentlyContinue)) { + $text = ([string]$line).Trim() + if ([string]::IsNullOrWhiteSpace($text)) { continue } + $port = 0 + # A malformed line is dropped rather than thrown on: this record is read + # on the start path, and a corrupt sidecar must not make the deploy + # unrunnable. The caller's ExpectedPorts still cover the current config. + if ([int]::TryParse($text, [ref]$port) -and $port -ge 1 -and $port -le 65535) { + if ($ports -notcontains $port) { $ports += $port } + } + } + return @($ports) +} + +function Get-HostNativeOrphanListener { + # Fail-closed answer to "is something OTHER than this service's recorded + # process tree still holding the ports this service owns?" - $null when the + # answer is no, a report object when it is yes. + # + # Phase 1 of the deploy already SAW the orphan in the failure this fixes + # ("port TCP/49150 occupied by our PID 188705 (kit.exe) - already running, + # will skip start") but that observation never reached the start decision: + # Phase 4c asked the pid file instead, the pid file had just been deleted as + # stale, and a second Kit was launched into the live one. This function is + # the observation the start decision can actually consume. + # + # "Accounted for" is deliberately narrow: the pid file must exist, its + # recorded process must still be alive, and the listener must be that + # process or one of its descendants. Everything else - no pid file, a dead + # recorded pid, an unrelated holder, or a listener whose owner the OS will + # not reveal (Get-PlatformTcpListenerPid returns -1) - is unaccounted and + # therefore a hard stop. Refusing is the safe direction: the caller can only + # ever choose between "refuse" and "start a second instance into a live one". + [CmdletBinding()] + param( + [Parameter(Mandatory = $true)][string] $Name, + [Parameter(Mandatory = $true)][string] $RunDir, + # The ports THIS run intends to use. Unioned with the recorded ports so a + # config change cannot hide an orphan that is still holding the previous + # instance's ports, and so a first-ever launch is covered too. + [AllowEmptyCollection()][int[]] $ExpectedPorts = @(), + [scriptblock] $PortLookupFn = { + param($port) + Get-PlatformTcpListenerPid -Port ([int]$port) + }, + [scriptblock] $GetProcessFn = { + param($procId) + try { Get-Process -Id $procId -ErrorAction Stop } catch { $null } + }, + [scriptblock] $ChildPidLookup = { + param($parentId) + @(Get-PlatformChildProcessIds -ParentProcessId ([int]$parentId)) + }, + # Phase 4c also reaches this gate straight after deliberately stopping + # the previous tree (runtime parameters changed). A force-killed process + # does not release its listening socket at the instant the stop call + # returns, and turning that teardown window into a hard deploy failure + # would trade one flaky outcome for another. So a holder must still be + # there after this budget before it counts. Zero means answer on the + # first observation. + [ValidateRange(0, 60000)][int] $SettleTimeoutMs = 3000, + [scriptblock] $SleepFn = { + param($milliseconds) + Start-Sleep -Milliseconds ([int]$milliseconds) + } + ) + + $recordedPorts = @(Get-HostNativeServiceListenPorts -Name $Name -RunDir $RunDir) + $ports = @(@(@($recordedPorts) + @($ExpectedPorts)) | + ForEach-Object { [int]$_ } | + Where-Object { $_ -ge 1 -and $_ -le 65535 } | + Sort-Object -Unique) + if ($ports.Count -eq 0) { return $null } + + $pidFile = Join-Path $RunDir "$Name.pid" + $budget = [System.Diagnostics.Stopwatch]::StartNew() + $accounted = @{} + $recordedProcessId = 0 + $orphanPorts = @() + $orphanProcessIds = @() + while ($true) { + # Descendants of a LIVE recorded pid are ours. A dead recorded pid is + # expanded from nothing on purpose: on Linux the kernel re-parents the + # orphan, so the ppid link back to a dead launcher is gone and expanding + # from it would silently claim whatever later inherited that pid. + # Re-read every pass: the tree can still be dying underneath us. + $accounted = @{} + $recordedProcessId = 0 + if (Test-Path -LiteralPath $pidFile -PathType Leaf) { + $raw = Get-Content -LiteralPath $pidFile -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($raw -and [int]::TryParse(([string]$raw).Trim(), [ref]$recordedProcessId)) { + if ($null -ne (& $GetProcessFn $recordedProcessId)) { + $stack = @([int]$recordedProcessId) + while ($stack.Count -gt 0) { + $current = [int]$stack[0] + $stack = @($stack | Select-Object -Skip 1) + if ($accounted.ContainsKey($current)) { continue } + $accounted[$current] = $true + # Enumeration failure is fatal here, exactly as it is in + # Stop-HostNativeService: a tree we cannot enumerate must + # never be used to clear a start decision. + $stack += @(& $ChildPidLookup $current) + } + } + } + } + + $orphanPorts = @() + $orphanProcessIds = @() + foreach ($port in $ports) { + $listenerProcessId = & $PortLookupFn ([int]$port) + if ($null -eq $listenerProcessId) { continue } + $listenerProcessId = [int]$listenerProcessId + if ($accounted.ContainsKey($listenerProcessId)) { continue } + $orphanPorts += [int]$port + if ($orphanProcessIds -notcontains $listenerProcessId) { $orphanProcessIds += $listenerProcessId } + } + if ($orphanPorts.Count -eq 0) { return $null } + if ($budget.ElapsedMilliseconds -ge $SettleTimeoutMs) { break } + & $SleepFn 250 + } + + return [pscustomobject]@{ + Name = $Name + Ports = @($orphanPorts) + ProcessIds = @($orphanProcessIds) + RecordedPorts = @($recordedPorts) + RecordedPid = [int]$recordedProcessId + AccountedPids = @(@($accounted.Keys) | ForEach-Object { [int]$_ } | Sort-Object) + } +} + function Remove-StalePidFile { + # Removes ONLY the pid file, and only when the pid it records is gone. + # + # It deliberately leaves the .ports sidecar in place (#640). Deleting + # it here would be the exact fail-open this repo already shipped: a dead + # launcher whose Kit child is still alive would lose its last trace, and the + # next Phase 4c would read "nothing is running" and start a second instance + # into the live one. Only a deliberate stop clears that record. [CmdletBinding()] param( [Parameter(Mandatory = $true)][string] $Name, @@ -209,6 +369,11 @@ function Stop-HostNativeService { ) $pidFile = Join-Path $RunDir "$Name.pid" $jobFile = Join-Path $RunDir "$Name.job" + # The port record is a claim on resources, so a DELIBERATE stop is what + # releases it (#640). Every return path below removes it: after this call the + # service is either gone or it threw, and in neither case may a later start + # decision keep treating the old claim as live. + $portFile = Join-Path $RunDir "$Name.ports" # Job-first: a recorded boundary makes the stop authoritative. Found+Proven # terminated the whole membership set; Found=$false is proven-dead by @@ -223,12 +388,16 @@ function Stop-HostNativeService { if ($jobReport.Supported) { Remove-Item -LiteralPath $jobFile -Force -ErrorAction SilentlyContinue Remove-Item -LiteralPath $pidFile -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $portFile -Force -ErrorAction SilentlyContinue return $true } } Remove-Item -LiteralPath $jobFile -Force -ErrorAction SilentlyContinue } + # The two $false returns below stopped NOTHING (no pid file, or a pid file we + # cannot parse), so they must not clear the port claim either - that is the + # orphan case, and erasing its last record here would recreate #640. if (-not (Test-Path -LiteralPath $pidFile)) { return $false } $raw = Get-Content -LiteralPath $pidFile -ErrorAction SilentlyContinue | Select-Object -First 1 $procId = 0 @@ -251,6 +420,7 @@ function Stop-HostNativeService { & $StopProcessFn ([int]$ids[$i]) } Remove-Item -LiteralPath $pidFile -Force -ErrorAction SilentlyContinue + Remove-Item -LiteralPath $portFile -Force -ErrorAction SilentlyContinue return $true } @@ -262,6 +432,10 @@ function Start-HostNativeService { [Parameter(Mandatory = $true)][string] $FilePath, [string[]] $ArgumentList = @(), [Parameter(Mandatory = $true)][string] $RunDir, + # TCP ports this service's process TREE will own. Recorded as a sidecar + # so liveness stops depending on the launcher pid alone (#640): the + # holder is usually a child, and the child outlives the pid file. + [AllowEmptyCollection()][int[]] $ListenPorts = @(), [ValidateSet('Hidden','Normal')] [string] $WindowStyle = 'Hidden', [ValidateRange(0, 30000)][int] $DetachTimeoutMs = 5000, [scriptblock] $DetachProbeFn = { @@ -293,6 +467,22 @@ function Start-HostNativeService { $errFile = "$logFile.err" $pidFile = Join-Path $RunDir "$Name.pid" $jobFile = Join-Path $RunDir "$Name.job" + $portFile = Join-Path $RunDir "$Name.ports" + + # Written BEFORE the launch, not after (#640). This record is a claim, not an + # observation: the moment Start-Process returns, a child may already be + # binding these ports, and a launch that then fails its detach check must + # still leave the claim behind for the next run's orphan preflight. Recording + # it afterwards would reopen the same window the pid file already has. + $declaredPorts = @(@($ListenPorts) | ForEach-Object { [int]$_ } | Where-Object { $_ -ge 1 -and $_ -le 65535 } | Sort-Object -Unique) + if ($declaredPorts.Count -gt 0) { + Set-Content -LiteralPath $portFile -Value ($declaredPorts -join [string][char]10) -Encoding ascii + } + else { + # A service that declares no ports must not inherit a previous launch's + # claim; a stale record would fail the next start closed for nothing. + Remove-Item -LiteralPath $portFile -Force -ErrorAction SilentlyContinue + } # Off Windows the service must OUTLIVE the session that started it: a remote # deploy runs over SSH, and a service that dies at disconnect makes the whole @@ -1028,10 +1218,15 @@ function Start-HostNativeKit { $arguments += ($SpectatorStreamPorts -join ',') } + # The signal ports are the observable identity of a running Kit tree (#640). + # They are TCP LISTEN sockets, so Get-PlatformTcpListenerPid can attribute + # them on both platforms; the WebRTC media ports are UDP and are deliberately + # not part of this record - one probe shape, one meaning. return (Start-HostNativeService ` -Name 'bim-streaming-server' ` -WorkingDirectory (Join-Path $RepoRoot 'bim-streaming-server') ` -FilePath (Get-HostNativePowerShellExe) ` -ArgumentList $arguments ` + -ListenPorts (@($SignalPort) + @($SpectatorSignalPorts)) ` -RunDir $runDir) } diff --git a/scripts/tests/test-deploy-governance-static.ps1 b/scripts/tests/test-deploy-governance-static.ps1 index bb9e98506..a7ec59993 100644 --- a/scripts/tests/test-deploy-governance-static.ps1 +++ b/scripts/tests/test-deploy-governance-static.ps1 @@ -441,6 +441,25 @@ if ($dockerFailureIndex -lt 0 -or $webPlaneSignatureSaveIndex -le $dockerFailure throw 'web-plane signature must be persisted only after docker compose succeeds' } +# #640: Phase 4c must consult the orphan gate BEFORE it launches a Kit, and it +# must refuse rather than adopt or race the holder. The observation that a +# previous instance is still holding the streaming ports already existed in +# Phase 1's audit output; what was missing was any path from that observation to +# the start decision, so a dead launcher plus a live Kit child produced a second +# Kit that deadlocked with no listener and no log of its own. +Assert-Contains $launcher 'function Get-HostNativeOrphanListener' 'host-native launcher must expose the orphaned-listener detector' +Assert-Contains $launcher 'function Get-HostNativeServiceListenPorts' 'host-native launcher must expose the recorded port claim' +Assert-Contains $deploy '$kitOrphan = Get-HostNativeOrphanListener' 'deploy.ps1 must run the orphaned-Kit gate on the start path' +Assert-Contains $deploy '-ExpectedPorts (@($resolvedKitSignalPort) + @($resolvedSpectatorSignalPorts))' 'the orphaned-Kit gate must cover every signal port this run intends to use' +Assert-Contains $deploy 'stage=4c Phase 4c refusing to start a second Kit' 'deploy.ps1 must fail closed instead of starting a second Kit' +Assert-Contains $deploy 'Stop it first with scripts/stop-all.ps1, then re-run this deploy' 'the orphaned-Kit refusal must name the executable remedy' +Assert-Contains $deploy "Print-FinalSummary -ExitCode 4 -FailedPhase 'Phase 4c (orphaned Kit holds the streaming ports)'" 'the orphaned-Kit refusal must exit through the Phase 4 failure summary' +$kitOrphanGateIndex = $deploy.IndexOf('$kitOrphan = Get-HostNativeOrphanListener') +$kitStartIndex = $deploy.IndexOf('$startInfo = Start-HostNativeKit') +if ($kitOrphanGateIndex -lt 0 -or $kitStartIndex -lt 0 -or $kitStartIndex -le $kitOrphanGateIndex) { + throw 'deploy.ps1 must evaluate the orphaned-Kit gate before Start-HostNativeKit, not after' +} + $kitBuildIndex = $deploy.IndexOf('Invoke-KitRepoBuild') $cadHardeningIndex = $deploy.IndexOf('harden-cad-extension-cache.py') $envMergeIndex = $deploy.IndexOf('# fix: .env / .env.example missing-key merge') diff --git a/scripts/tests/test-host-native-launcher.ps1 b/scripts/tests/test-host-native-launcher.ps1 index e86948d1c..930f46257 100644 --- a/scripts/tests/test-host-native-launcher.ps1 +++ b/scripts/tests/test-host-native-launcher.ps1 @@ -1736,4 +1736,252 @@ finally { Remove-TestSandbox -Path $conversionProbeSandbox } +# --------------------------------------------------------------------------- +# #640: an orphaned Kit child is invisible to pid-file liveness, so Phase 4c +# started a second instance into a live one. Two halves are proven here: +# (1) a launch RECORDS the ports its process tree will own, and that record +# outlives the pid file that Remove-StalePidFile is right to delete; +# (2) Get-HostNativeOrphanListener turns that record into a refusal signal. +# +# Everything below runs on fixtures and injected probes. No Kit is launched, no +# real port in 49100-49110 (or any other real port) is bound, and the only real +# processes started are short-lived Python sleeps that bind nothing. +# --------------------------------------------------------------------------- + +# Test O1: Start-HostNativeService records the declared ports as a sidecar, and +# a launch that declares none clears a previous claim instead of inheriting it. +. $modulePath +$portRecordSandbox = New-TestSandbox -Prefix 'hn-port-record' +try { + $portRunDir = Join-Path $portRecordSandbox 'scripts\.run' + New-Item -ItemType Directory -Path $portRunDir -Force | Out-Null + $portProbePython = Resolve-PlatformSystemPython + # Supported=$false keeps the launch off real Job Objects; the other keys are + # present only because the parameter contract requires the whole table. + $noBoundaryOps = @{ + Supported = { $false } + Create = { param($jobName) throw 'unsupported boundary must never create' } + Assign = { param($handle, $childId) throw 'unsupported boundary must never assign' } + Anchor = { param($handle, $childId) throw 'unsupported boundary must never anchor' } + Terminate = { param($handle) throw 'unsupported boundary must never terminate' } + Close = { param($handle) throw 'unsupported boundary must never close' } + } + $portedInfo = Start-HostNativeService ` + -Name 'ported-service' ` + -WorkingDirectory $portRecordSandbox ` + -FilePath $portProbePython ` + -ArgumentList @('-c', 'import time; time.sleep(60)') ` + -RunDir $portRunDir ` + -ListenPorts @(49150, 49100, 49100) ` + -DetachProbeFn { param($processId) $true } ` + -JobBoundaryOps $noBoundaryOps + try { + $portSidecar = Join-Path $portRunDir 'ported-service.ports' + Assert-True (Test-Path -LiteralPath $portSidecar -PathType Leaf) 'a launch that declares ports records the sidecar' + Assert-Equal '49100,49150' ((Get-HostNativeServiceListenPorts -Name 'ported-service' -RunDir $portRunDir) -join ',') 'the recorded claim is de-duplicated and sorted' + } + finally { + Stop-Process -Id $portedInfo.Pid -Force -ErrorAction SilentlyContinue + } + + # A service that declares no ports must not inherit the previous claim. + $portlessInfo = Start-HostNativeService ` + -Name 'ported-service' ` + -WorkingDirectory $portRecordSandbox ` + -FilePath $portProbePython ` + -ArgumentList @('-c', 'import time; time.sleep(60)') ` + -RunDir $portRunDir ` + -DetachProbeFn { param($processId) $true } ` + -JobBoundaryOps $noBoundaryOps + try { + Assert-True (-not (Test-Path -LiteralPath (Join-Path $portRunDir 'ported-service.ports'))) 'a portless launch clears the previous port claim' + Assert-Equal '' ((Get-HostNativeServiceListenPorts -Name 'ported-service' -RunDir $portRunDir) -join ',') 'no sidecar reads back as no recorded ports' + } + finally { + Stop-Process -Id $portlessInfo.Pid -Force -ErrorAction SilentlyContinue + } + Write-TestPass 'host-native launch records the ports its tree will own (#640)' +} +finally { + Remove-TestSandbox -Path $portRecordSandbox +} + +# Test O2: Remove-StalePidFile still removes the stale pid file, and deliberately +# leaves the port claim behind - that record is the only remaining trace of an +# orphaned child, and deleting it here is exactly how #640 lost the orphan. +$staleRecordSandbox = New-TestSandbox -Prefix 'hn-stale-ports' +try { + $staleRunDir = Join-Path $staleRecordSandbox 'scripts\.run' + New-Item -ItemType Directory -Path $staleRunDir -Force | Out-Null + Set-Content -LiteralPath (Join-Path $staleRunDir 'bim-streaming-server.pid') -Value '216268' + Set-Content -LiteralPath (Join-Path $staleRunDir 'bim-streaming-server.ports') -Value "49100`n49150" + $removed = Remove-StalePidFile -Name 'bim-streaming-server' -RunDir $staleRunDir -GetProcessFn { param($procId) $null } + Assert-True $removed 'a dead recorded pid is still cleaned up' + Assert-True (-not (Test-Path -LiteralPath (Join-Path $staleRunDir 'bim-streaming-server.pid'))) 'the stale pid file is removed' + Assert-Equal '49100,49150' ((Get-HostNativeServiceListenPorts -Name 'bim-streaming-server' -RunDir $staleRunDir) -join ',') 'the port claim survives stale-pid cleanup' + Write-TestPass 'stale-pid cleanup keeps the port claim that outlives the launcher (#640)' +} +finally { + Remove-TestSandbox -Path $staleRecordSandbox +} + +# Test O3: Get-HostNativeOrphanListener - the start decision matrix. +$orphanSandbox = New-TestSandbox -Prefix 'hn-orphan-listener' +try { + $orphanRunDir = Join-Path $orphanSandbox 'scripts\.run' + New-Item -ItemType Directory -Path $orphanRunDir -Force | Out-Null + Set-Content -LiteralPath (Join-Path $orphanRunDir 'bim-streaming-server.ports') -Value "49100`n49150" + # The measured #640 shape: launcher 216268 gone, orphaned Kit 216306 still + # holding a spectator signal port, pid file already deleted as stale. + $orphanPortOwners = @{ 49100 = $null; 49150 = 216306 } + $orphanProbe = { param($port) $orphanPortOwners[[int]$port] } + $noProcess = { param($procId) $null } + $noChildren = { param($parentId) @() } + # SettleTimeoutMs 0 keeps every "should report" case to a single observation; + # the settle window itself is proven separately at the end of this block. + $now = @{ SettleTimeoutMs = 0; SleepFn = { param($milliseconds) } } + + $reported = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -PortLookupFn $orphanProbe -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-True ($null -ne $reported) 'a surviving holder with no pid file is reported' + Assert-Equal '49150' (@($reported.Ports) -join ',') 'the report names the port that is actually held' + Assert-Equal '216306' (@($reported.ProcessIds) -join ',') 'the report names the holding pid, not the recorded launcher pid' + Assert-Equal '49100,49150' (@($reported.RecordedPorts) -join ',') 'the report carries the claim it checked' + + # Same answer when the pid file still exists but its process is gone: that is + # the window between the launcher dying and Phase 2 deleting the pid file. + Set-Content -LiteralPath (Join-Path $orphanRunDir 'bim-streaming-server.pid') -Value '216268' + $deadRecorded = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -PortLookupFn $orphanProbe -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-True ($null -ne $deadRecorded) 'a dead recorded pid cannot account for a live holder' + Assert-Equal '216306' (@($deadRecorded.ProcessIds) -join ',') 'the dead launcher pid is not treated as the holder' + + # Healthy idempotent re-run: the recorded launcher is alive and the holder is + # its child, so nothing is reported and Phase 4c keeps its existing skip path. + $liveTree = { param($parentId) if ([int]$parentId -eq 216268) { @(216306) } else { @() } } + $liveProcess = { param($procId) @{ Id = [int]$procId } } + $healthy = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -PortLookupFn $orphanProbe -GetProcessFn $liveProcess -ChildPidLookup $liveTree + Assert-True ($null -eq $healthy) 'a live launcher accounts for its own child' + + # A live launcher does NOT account for an unrelated holder: two instances. + $strangerOwners = @{ 49100 = $null; 49150 = 188705 } + $twoInstances = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -PortLookupFn { param($port) $strangerOwners[[int]$port] } -GetProcessFn $liveProcess -ChildPidLookup $liveTree + Assert-True ($null -ne $twoInstances) 'a holder outside our tree is reported even when our tree is alive' + Assert-Equal '188705' (@($twoInstances.ProcessIds) -join ',') 'the unrelated holder is named' + + # Get-PlatformTcpListenerPid returns -1 for "occupied, owner not visible". + # Unknown ownership must fail closed, never read as free. + $invisibleOwners = @{ 49100 = -1; 49150 = $null } + $invisible = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -PortLookupFn { param($port) $invisibleOwners[[int]$port] } -GetProcessFn $liveProcess -ChildPidLookup $liveTree + Assert-True ($null -ne $invisible) 'an occupied port with an invisible owner fails closed' + Assert-Equal '49100' (@($invisible.Ports) -join ',') 'the invisible-owner port is the one reported' + Assert-Equal '-1' (@($invisible.ProcessIds) -join ',') 'the sentinel owner is surfaced rather than swallowed' + + # Free ports mean no report, whatever the claim says. + $allFree = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -PortLookupFn { param($port) $null } -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-True ($null -eq $allFree) 'a recorded claim with every port free is not an orphan' + + # The probe set is the UNION of the recorded claim and this run's expectation, + # so neither a config change nor a first-ever launch can hide a holder. + $script:probedPorts = @() + $unionProbe = { + param($port) + $script:probedPorts += [int]$port + if ([int]$port -eq 49101) { return 4242 } + return $null + } + $union = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir @now ` + -ExpectedPorts @(49101) -PortLookupFn $unionProbe -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-Equal '49100,49101,49150' (@($script:probedPorts | Sort-Object -Unique) -join ',') 'recorded and expected ports are both probed' + Assert-True ($null -ne $union) 'a holder on a port only this run expects is still reported' + Assert-Equal '49101' (@($union.Ports) -join ',') 'the newly expected port is the one reported' + + # Nothing recorded and nothing expected: no probe, no verdict, no refusal. + $script:probedPorts = @() + $emptyRunDir = Join-Path $orphanSandbox 'empty-run' + New-Item -ItemType Directory -Path $emptyRunDir -Force | Out-Null + $nothing = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $emptyRunDir @now ` + -PortLookupFn $unionProbe -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-True ($null -eq $nothing) 'no claim and no expectation is not an orphan' + Assert-Equal 0 @($script:probedPorts).Count 'with no ports to check the detector probes nothing' + + # Settle window: Phase 4c also reaches this gate immediately after stopping + # the previous tree itself, and a force-killed listener needs a moment to + # release its socket. A holder that is gone on a later pass is teardown, not + # an orphan; one still there when the budget runs out is an orphan. + $script:teardownProbeCalls = 0 + $script:sleepCalls = 0 + $noSleep = { param($milliseconds) $script:sleepCalls++ } + $teardownProbe = { + param($port) + $script:teardownProbeCalls++ + if ($script:teardownProbeCalls -le 2) { return 999001 } + return $null + } + $settled = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir ` + -SettleTimeoutMs 5000 -SleepFn $noSleep ` + -PortLookupFn $teardownProbe -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-True ($null -eq $settled) 'a listener that disappears within the settle budget is teardown, not an orphan' + Assert-True ($script:sleepCalls -ge 1) 'the settle window actually waits before re-observing' + + $script:sleepCalls = 0 + $persistent = Get-HostNativeOrphanListener -Name 'bim-streaming-server' -RunDir $orphanRunDir ` + -SettleTimeoutMs 5000 -SleepFn $noSleep ` + -PortLookupFn $orphanProbe -GetProcessFn $noProcess -ChildPidLookup $noChildren + Assert-True ($null -ne $persistent) 'a holder that survives the settle budget is still reported' + Assert-Equal '49150' (@($persistent.Ports) -join ',') 'the surviving holder keeps its port in the report' + Write-TestPass 'orphaned listener detection is fail-closed and tree-aware (#640)' +} +finally { + Remove-TestSandbox -Path $orphanSandbox +} + +# Test O4: a deliberate stop releases the claim; a stop that stopped nothing does +# not - otherwise the recovery path would erase the very record it needs. +$stopClaimSandbox = New-TestSandbox -Prefix 'hn-stop-ports' +try { + $stopClaimRunDir = Join-Path $stopClaimSandbox 'scripts\.run' + New-Item -ItemType Directory -Path $stopClaimRunDir -Force | Out-Null + + Set-Content -LiteralPath (Join-Path $stopClaimRunDir 'svc.pid') -Value '4242' + Set-Content -LiteralPath (Join-Path $stopClaimRunDir 'svc.job') -Value 'Local\aibim-job-svc' + Set-Content -LiteralPath (Join-Path $stopClaimRunDir 'svc.ports') -Value '49100' + $null = Stop-HostNativeService -Name 'svc' -RunDir $stopClaimRunDir ` + -ChildPidLookup { param($parentId) @() } ` + -StopProcessFn { param($procId) } ` + -JobStopFn { param($jobName) [pscustomobject]@{ Found = $true; MemberPids = @(4242); Proven = $true; Supported = $true } } + Assert-True (-not (Test-Path -LiteralPath (Join-Path $stopClaimRunDir 'svc.ports'))) 'the job-first stop releases the port claim' + + Set-Content -LiteralPath (Join-Path $stopClaimRunDir 'svc2.pid') -Value '4343' + Set-Content -LiteralPath (Join-Path $stopClaimRunDir 'svc2.ports') -Value '49150' + $null = Stop-HostNativeService -Name 'svc2' -RunDir $stopClaimRunDir ` + -ChildPidLookup { param($parentId) @() } ` + -StopProcessFn { param($procId) } ` + -JobStopFn { param($jobName) [pscustomobject]@{ Found = $false; MemberPids = @(); Proven = $false; Supported = $false } } + Assert-True (-not (Test-Path -LiteralPath (Join-Path $stopClaimRunDir 'svc2.ports'))) 'the legacy walk releases the port claim too' + + Set-Content -LiteralPath (Join-Path $stopClaimRunDir 'svc3.ports') -Value '49160' + $stoppedNothing = Stop-HostNativeService -Name 'svc3' -RunDir $stopClaimRunDir ` + -ChildPidLookup { param($parentId) @() } ` + -StopProcessFn { param($procId) } ` + -JobStopFn { param($jobName) [pscustomobject]@{ Found = $false; MemberPids = @(); Proven = $false; Supported = $false } } + Assert-True (-not $stoppedNothing) 'a stop with no pid file reports that it stopped nothing' + Assert-Equal '49160' ((Get-HostNativeServiceListenPorts -Name 'svc3' -RunDir $stopClaimRunDir) -join ',') 'a stop that stopped nothing keeps the claim' + Write-TestPass 'only a stop that terminated something releases the port claim (#640)' +} +finally { + Remove-TestSandbox -Path $stopClaimSandbox +} + +# Test O5: the Kit launcher declares the TCP signal ports as its claim. Media +# ports are UDP and stay out of a record that only a TCP probe can attribute. +$launcherBody = Get-Content -LiteralPath $modulePath -Raw +Assert-True ($launcherBody -match '-ListenPorts \(@\(\$SignalPort\) \+ @\(\$SpectatorSignalPorts\)\)') 'Start-HostNativeKit declares its signal ports as the recorded claim' +Assert-True (-not ($launcherBody -match '-ListenPorts.*\$StreamPort')) 'the UDP media port is not recorded as a TCP claim' +Write-TestPass 'Kit launch declares the signal ports it will own (#640)' + Write-Host "`n=== test-host-native-launcher.ps1: ALL PASSED ===" -ForegroundColor Green From eba8ca81c05e2f42b34654ffb3d95a1e338e96ad Mon Sep 17 00:00:00 2001 From: monkey1sai <26239865+monkey1sai@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:46:45 +0800 Subject: [PATCH 2/2] chore(governance): open the orphan-kit-liveness-preflight bootstrap entry (#640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/deploy.ps1 and scripts/lib/host-native-launcher.ps1 are both classified verification-mechanism paths, and this PR changes what "already running / safe to start" means on the canonical deploy path. Per docs/agents/self-referential-bootstrap.md §2.1 class 2, the deploy contract only ever rebuilds and verifies already-merged origin/main content, so the changed behaviour - a port claim written by a live launch, and a Phase 4c refusal that only fires against a real orphaned Kit holding a real LISTEN socket - cannot be proven by the canonical mechanism before this change reaches origin/main. Same shape as PR #647. Fixture-driven unit tests prove the detection and refusal logic; they do not replace a canonical deployment cycle, which is exactly what the fixpoint closure will re-run. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01GSyeoncjEm8rcfRDgnS6CB --- .../self-referential-bootstrap/README.md | 48 +++++++++++++++++++ .../verification.txt | 44 +++++++++++++++++ .../self-referential-bootstrap-ledger.json | 28 +++++++++++ 3 files changed, 120 insertions(+) create mode 100644 docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/README.md create mode 100644 docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/verification.txt diff --git a/docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/README.md b/docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/README.md new file mode 100644 index 000000000..9c71f4882 --- /dev/null +++ b/docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/README.md @@ -0,0 +1,48 @@ +# orphan-kit-liveness-preflight — self-referential bootstrap + +- `stack_kind=self_referential_bootstrap` +- Originating PR: `#653` +- Ledger entry: `orphan-kit-liveness-preflight` +- Opening base: `d72f3a66e738a8f1654707185cc14d836dd26759` +- Pre-opening branch head: `449e6eb0f3ca99dd8bcaad219ec08e3b68e53ece` +- Verification contract: `orphan-kit-liveness-preflight/v1` +- Contract SHA-256: `aaf83a0022155d2b8858e98570bbf15ead369b298ce8360402982a6e0c149223` +- Mechanism paths: `scripts/deploy.ps1`, `scripts/lib/host-native-launcher.ps1`, `scripts/self-referential-bootstrap-ledger.json` + +本 PR 改的是 canonical deploy path 上「已經有實例在跑嗎、可以啟動嗎」這個判定 +本身(issue #640)。兩層變更各自只在真實部署當下才存在： + +1. `Start-HostNativeService` 新寫的 `.ports` claim，只有一次真的 + `Start-Process` 啟動才會產生；`Remove-StalePidFile` 刻意不刪它，也只有在真實 + 部署把 launcher 弄死、Kit 子行程存活時才看得出差別。 +2. `Get-HostNativeOrphanListener` 的拒絕，只有在真的有一個孤兒 Kit 佔著真的 + LISTEN socket 時才會觸發；Phase 4c 的 exit 4 也只有在真實部署流程裡才會走到。 + +依 `docs/agents/self-referential-bootstrap.md` §2.1 第 2 類，部署契約只重建／驗證 +已 merge 的 `origin/main` 內容(`scripts/deploy.ps1` 與 +`scripts/lib/host-native-launcher.ps1` 皆列於 `Get-SelfReferentialMechanismPaths`)， +因此在本變更抵達 origin/main 之前，無法用正規機制對「變更後行為」取證——不是 +「報告格式沒有前版可比」那種 §2.1 明確排除的情況，而是契約本身禁止在 merge 前 +用 canonical deployment 驗證變更後的啟動決策。同一形狀的既有實例為 PR #647 +(`remote-deploy-tag-origin-main-sync`)。 + +單元測試能證明什麼、不能證明什麼，寫清楚：能證明的是 detection 與 refusal 的 +邏輯——port claim 的寫入／保留／釋放、union 探測、live-tree 歸屬、 +owner-not-visible(-1) fail-closed、settle window、以及 deploy.ps1 Phase 4c 在 +`Start-HostNativeKit` 之前就評估這道閘門且以 exit 4 收尾。不能證明的是真實 +canonical-linux 上「launcher 死掉、Kit 子行程仍活著佔埠」這個狀態下的端到端行為； +fixture 造出來的孤兒不等於一次真實部署週期。這正是本 debt 存在的理由，關帳留給 +merge 後、以變更後機制重跑本 contract 五道命令的 ledger-only fixpoint PR。 + +本 evidence 只記錄 bootstrap opening，不是 canonical post-merge evidence，也不是 +fixpoint evidence，不得用來關閉本 entry。 + +GitNexus：`gitnexus impact Get-HostNativeOrphanListener -d upstream -r +AI-BIM-governance` 回報 `Target not found`(`impactedCount=0`, `risk=UNKNOWN`)。 +與 PR #647 記載的成因相同——GitNexus 目前不抽取 PowerShell 函式層級符號(`.ps1` +只以 File node 索引)，屬工具涵蓋缺口而非索引過期，故記為 unavailable，不冒充 +pass。替代證據為手動盤點的呼叫點：`Get-HostNativeOrphanListener` 只有兩個 +consumer(`scripts/deploy.ps1` Phase 4c 與 +`scripts/tests/test-host-native-launcher.ps1`)；`.ports` sidecar 的 +reader/writer 全數列舉為 `Start-HostNativeService`、`Stop-HostNativeService`、 +`Get-HostNativeServiceListenPorts`、`Remove-StalePidFile`(刻意不動)。 diff --git a/docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/verification.txt b/docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/verification.txt new file mode 100644 index 000000000..1dc0b7fc0 --- /dev/null +++ b/docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/verification.txt @@ -0,0 +1,44 @@ +stack_kind=self_referential_bootstrap +pr=653 +ledger_entry=orphan-kit-liveness-preflight +opening_base=d72f3a66e738a8f1654707185cc14d836dd26759 +pre_opening_head=449e6eb0f3ca99dd8bcaad219ec08e3b68e53ece +verification_contract=orphan-kit-liveness-preflight/v1 +contract_sha256=aaf83a0022155d2b8858e98570bbf15ead369b298ce8360402982a6e0c149223 +mechanism_paths=scripts/deploy.ps1,scripts/lib/host-native-launcher.ps1,scripts/self-referential-bootstrap-ledger.json + +status=bootstrap_contract_pass +verified_at=2026-08-19T10:44:44Z +subject=working tree based on pre-opening head 449e6eb0f3ca99dd8bcaad219ec08e3b68e53ece with the exact ledger and evidence additions listed by this PR + +test-host-native-launcher +command=pwsh -NoProfile -NonInteractive -File scripts/tests/test-host-native-launcher.ps1 +result=exit 0; ALL PASSED, including the five new #640 cases (port claim recorded by a real launch; stale-pid cleanup keeps the claim; orphan detection matrix incl. dead recorded pid, live-tree attribution, unrelated holder, owner-not-visible sentinel, union probing and the settle window; only a stop that terminated something releases the claim; Kit launch declares its signal ports) + +test-deploy-governance-static +command=pwsh -NoProfile -NonInteractive -File scripts/tests/test-deploy-governance-static.ps1 +result=exit 0; PASS deploy governance static checks, including the new assertions that Phase 4c evaluates Get-HostNativeOrphanListener BEFORE Start-HostNativeKit, refuses with stage=4c, names scripts/stop-all.ps1, and exits through Print-FinalSummary -ExitCode 4 + +test-self-referential-bootstrap +command=pwsh -NoProfile -NonInteractive -File scripts/tests/test-self-referential-bootstrap.ps1 +result=see PR body Validation section for the exact-head run + +test-pr-body-evidence +command=pwsh -NoProfile -NonInteractive -File scripts/tests/test-pr-body-evidence.ps1 +result=see PR body Validation section for the exact-head run + +invoke-powershell-static +command=pwsh -NoProfile -NonInteractive -File scripts/tests/invoke-powershell-static.ps1 +result=exit 0; [invoke-powershell-static] passed + +gitnexus-impact +command=gitnexus impact Get-HostNativeOrphanListener -d upstream -r AI-BIM-governance +result=UNKNOWN/unavailable; PowerShell function-level symbols are not indexed by GitNexus (.ps1 files are File nodes only), the same coverage gap recorded for PR #647. Recorded as unavailable, not as a pass. Substitute evidence: manual call-site inventory - Get-HostNativeOrphanListener has exactly two consumers (scripts/deploy.ps1 Phase 4c, scripts/tests/test-host-native-launcher.ps1), and every reader/writer of the .ports sidecar is enumerated in the README next to this file. + +windows-verification +tier=deploy_dryrun +result=scripts/deploy.ps1 -DryRun exits 0 on this Windows host; the exact-final-head GitHub Actions run URL is recorded in the PR body after the final push, not claimed by this pre-final-head bootstrap file. No live deployment, no service start, no port bind, and no Kit/GPU/WebRTC/browser evidence is claimed here. + +No credential value was read or emitted. No approval, merge, runtime stop, +deployment mutation, or production action was performed while opening this +debt. diff --git a/scripts/self-referential-bootstrap-ledger.json b/scripts/self-referential-bootstrap-ledger.json index e81ee0d50..09cd14248 100644 --- a/scripts/self-referential-bootstrap-ledger.json +++ b/scripts/self-referential-bootstrap-ledger.json @@ -840,6 +840,34 @@ "docs/evidence/remote-deploy-tag-origin-main-sync/fixpoint/summary.md" ] } + }, + { + "id": "orphan-kit-liveness-preflight", + "status": "open", + "pr": 653, + "opened_at": "2026-08-19T10:44:44Z", + "reason": "the canonical deploy path decides \"is an instance already running, and is it safe to start one\" only while a real deployment is executing against already-merged origin/main content, and the behaviour this PR changes exists only in that moment: the .ports claim is written by a live Start-HostNativeService, and the Phase 4c refusal only fires against a real orphaned Kit child that is still holding a real LISTEN socket after its launcher died. Neither state can be produced on a branch by the canonical mechanism before this change reaches origin/main; the unit suite proves the detection and refusal logic with injected port, process and child-enumeration probes and with sidecar fixtures, but a fixture orphan is not a canonical deployment cycle.", + "verification_mechanism_paths": [ + "scripts/deploy.ps1", + "scripts/lib/host-native-launcher.ps1", + "scripts/self-referential-bootstrap-ledger.json" + ], + "verification_contract": { + "id": "orphan-kit-liveness-preflight/v1", + "command_ids": [ + "test-host-native-launcher", + "test-deploy-governance-static", + "test-self-referential-bootstrap", + "test-pr-body-evidence", + "invoke-powershell-static" + ], + "contract_sha256": "aaf83a0022155d2b8858e98570bbf15ead369b298ce8360402982a6e0c149223" + }, + "bootstrap_evidence_refs": [ + "docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/README.md", + "docs/evidence/orphan-kit-liveness-preflight/self-referential-bootstrap/verification.txt" + ], + "fixpoint": null } ] }