Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
59 commits
Select commit Hold shift + click to select a range
0abbe97
fix(state): make lifecycle lock timeouts side-effect free
HOYALIM Jul 30, 2026
19f17ff
test(state): keep lifecycle deadline proof linear
HOYALIM Jul 30, 2026
de98a7d
fix(state): restore stale locks after deadline
HOYALIM Aug 3, 2026
9102bfd
fix(state): avoid reclaiming lock directories
HOYALIM Aug 3, 2026
21fc856
Merge branch 'main' into codex/mcp-lock-deadline
prekshivyas Aug 4, 2026
e495e8d
test(state): add lifecycle issue references
prekshivyas Aug 4, 2026
ce22702
fix(state): await stale lock reclamation
HOYALIM Aug 4, 2026
67ad43b
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
50ae472
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
1a2f6c5
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
5e39965
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
ea770d9
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
d597df5
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
046b530
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 4, 2026
3c00767
merge(main): refresh lifecycle lock deadline contract
HOYALIM Aug 4, 2026
c142857
test: keep lifecycle lock timing setup linear
HOYALIM Aug 4, 2026
96af782
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 5, 2026
8a33683
test(shields): isolate deadline snapshot reuse
jyaunches Aug 5, 2026
0071e88
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 5, 2026
2123889
fix(lock): await stale generation reclamation
jyaunches Aug 5, 2026
ab76f11
merge: refresh PR branch
jyaunches Aug 5, 2026
3b1ebc3
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 5, 2026
35138fa
test(e2e): wait for gateway fixture readiness
jyaunches Aug 5, 2026
33b70bf
test(lock): make corrupt containment deterministic
jyaunches Aug 5, 2026
782d16a
test(state): make lifecycle timing deterministic
HOYALIM Aug 5, 2026
81adc1a
merge(main): refresh lifecycle lock deadline contract
HOYALIM Aug 5, 2026
7a3bfc8
merge(main): refresh lifecycle lock timeout fix
prekshivyas Aug 5, 2026
4c8103a
test(state): make legacy lock recovery deterministic
prekshivyas Aug 5, 2026
956a4b2
merge(main): refresh lifecycle lock timeout fix
prekshivyas Aug 5, 2026
78b75c1
merge(main): refresh lifecycle lock timeout fix
prekshivyas Aug 5, 2026
45e311e
merge(main): refresh lifecycle lock timeout fix
prekshivyas Aug 5, 2026
cf076ce
fix(state): enforce sync lock deadlines
prekshivyas Aug 5, 2026
3b40c90
test(state): keep sync deadline tests linear
prekshivyas Aug 5, 2026
f59ebae
merge(main): refresh lifecycle lock timeout fix
prekshivyas Aug 5, 2026
2847f8c
merge(main): refresh lifecycle lock timeout fix
cv Aug 5, 2026
ebefae8
merge: refresh PR branch
jyaunches Aug 5, 2026
125dbba
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 5, 2026
2627e0e
Merge branch 'main' into codex/mcp-lock-deadline
jyaunches Aug 5, 2026
6c8a0b5
merge(main): refresh lifecycle lock timeout fix
cjagwani Aug 5, 2026
9cca0f3
test(state): stabilize lifecycle lock timing budget
cjagwani Aug 5, 2026
0ae9285
merge(main): refresh lifecycle lock timeout fix
cjagwani Aug 5, 2026
b9c72f4
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
c087024
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
d12c452
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
6998bfb
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
a14f566
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
95967ea
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
cd31fa8
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
2930357
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
c7645de
merge(pr): refresh #7859 onto current main
cjagwani Aug 5, 2026
d2e1fbd
fix(state): enforce lock acquisition deadline
cv Aug 5, 2026
3d258c6
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
25e7ae6
merge(main): refresh PR #7859
cv Aug 5, 2026
d9f3229
merge(main): refresh PR #7859
cjagwani Aug 5, 2026
52f9c8a
merge(pr): reconcile PR #7859 contributor updates
cv Aug 5, 2026
23d9fc2
docs(state): clarify lifecycle lock wording
cv Aug 5, 2026
e9abda8
merge(main): refresh PR #7859 remediation
cv Aug 5, 2026
a7ace6f
merge(main): reconcile PR #7859 with current main
cv Aug 5, 2026
4f8e3b8
test(state): extract lifecycle lock deadline clocks
cv Aug 5, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 62 additions & 21 deletions src/lib/state/mcp-lifecycle-lock-acquisition.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ export interface McpLifecycleLockOptions {
pollIntervalMs?: number;
timeoutMs?: number;
corruptLockGraceMs?: number;
/** Monotonic clock override used by deterministic deadline tests. */
monotonicNow?: () => number;
}

interface HeldLockLease {
Expand Down Expand Up @@ -72,10 +74,13 @@ function classifyObservedMcpLifecycleLock(
sandboxName: string,
corruptLockGraceMs: number,
corruptTracker: CorruptGenerationTracker,
now: number,
): McpLifecycleLockDisposition {
if (!observation.owner || observation.owner.sandboxName !== sandboxName) {
if (
(!observation.owner || observation.owner.sandboxName !== sandboxName) &&
observation.reclaimable
) {
const generation = `${observation.dev}:${observation.ino}:${observation.mtimeMs}`;
const now = performance.now();
if (corruptTracker.generation !== generation) {
corruptTracker.generation = generation;
corruptTracker.firstSeenAt = now;
Expand All @@ -85,36 +90,42 @@ function classifyObservedMcpLifecycleLock(
}
resetCorruptGenerationTracker(corruptTracker);
// The wall-clock arguments are irrelevant for a structurally valid owner.
return classifyMcpLifecycleLock(
observation,
sandboxName,
observation.mtimeMs,
corruptLockGraceMs,
);
return observation.owner === null
? "wait"
: classifyMcpLifecycleLock(observation, sandboxName, observation.mtimeMs, corruptLockGraceMs);
}

async function tryReapStaleLock(
lockPath: string,
sandboxName: string,
corruptLockGraceMs: number,
corruptTracker: CorruptGenerationTracker,
monotonicNow: () => number,
assertBeforeDeadline: () => void,
): Promise<boolean> {
const reaperPath = `${lockPath}.reaper`;
const reaperToken = crypto.randomUUID();
const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken);
assertBeforeDeadline();
if (!(await writeMcpLifecycleLockCandidateAndLink(reaperPath, reaperOwner))) return false;

try {
const latest = await readMcpLifecycleLockObservation(lockPath);
if (!latest) return true;
if (
classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !==
"stale"
classifyObservedMcpLifecycleLock(
latest,
sandboxName,
corruptLockGraceMs,
corruptTracker,
monotonicNow(),
) !== "stale"
) {
return false;
}

return reclaimStaleMcpLifecycleLockGeneration(lockPath, latest);
assertBeforeDeadline();
return await reclaimStaleMcpLifecycleLockGeneration(lockPath, latest, assertBeforeDeadline);
} finally {
await safelyReleaseMcpLifecycleLock(reaperPath, reaperToken);
}
Expand All @@ -130,23 +141,27 @@ async function acquireMcpLifecycleLock(
options.corruptLockGraceMs,
DEFAULT_CORRUPT_LOCK_GRACE_MS,
);
const monotonicNow = options.monotonicNow ?? (() => performance.now());
const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir);
await fs.promises.mkdir(path.dirname(lockPath), {
recursive: true,
mode: 0o700,
});

const startedAt = performance.now();
const deadline = monotonicNow() + timeoutMs;
const corruptMainTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 };
const corruptReaperTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 };
let lastOwnerPid: number | null = null;
const assertBeforeDeadline = () => {
if (monotonicNow() < deadline) return;
const ownerSuffix = lastOwnerPid ? ` (owner pid ${lastOwnerPid})` : "";
throw new Error(
`Timed out waiting for the sandbox mutation lock for '${sandboxName}'${ownerSuffix}. Another lifecycle, policy, channel, shields, or snapshot operation is still running.`,
);
};

for (;;) {
if (performance.now() - startedAt >= timeoutMs) {
const ownerSuffix = lastOwnerPid ? ` (owner pid ${lastOwnerPid})` : "";
throw new Error(
`Timed out waiting for the sandbox mutation lock for '${sandboxName}'${ownerSuffix}. Another lifecycle, policy, channel, shields, or snapshot operation is still running.`,
);
}
assertBeforeDeadline();

const reaperPath = `${lockPath}.reaper`;
const reaperObservation = await readMcpLifecycleLockObservation(reaperPath);
Expand All @@ -156,12 +171,18 @@ async function acquireMcpLifecycleLock(
sandboxName,
corruptLockGraceMs,
corruptReaperTracker,
monotonicNow(),
);
if (reaperDisposition === "stale") {
// The reaper has the same atomic, PID-identified owner format as the
// main lock. A SIGKILL at any point in stale-lock cleanup is therefore
// recoverable without age-expiring a legitimate long operation.
await reclaimStaleMcpLifecycleLockGeneration(reaperPath, reaperObservation);
assertBeforeDeadline();
await reclaimStaleMcpLifecycleLockGeneration(
reaperPath,
reaperObservation,
assertBeforeDeadline,
);
continue;
}
await sleep(pollIntervalMs);
Expand All @@ -170,13 +191,22 @@ async function acquireMcpLifecycleLock(
resetCorruptGenerationTracker(corruptReaperTracker);

if (!(await mcpLifecycleLockPathExists(reaperPath))) {
assertBeforeDeadline();
const token = crypto.randomUUID();
const owner = createMcpLifecycleLockOwner(sandboxName, token);
if (await writeMcpLifecycleLockCandidateAndLink(lockPath, owner)) {
// A stale-lock reaper may have appeared between our pre-check and the
// atomic link. Do not enter the critical section until that generation
// gate has gone away.
if (!(await mcpLifecycleLockPathExists(reaperPath))) return { lockPath, token };
if (!(await mcpLifecycleLockPathExists(reaperPath))) {
try {
assertBeforeDeadline();
} catch (error) {
await safelyReleaseMcpLifecycleLock(lockPath, token);
throw error;
}
return { lockPath, token };
}
await safelyReleaseMcpLifecycleLock(lockPath, token);
}
}
Expand All @@ -190,9 +220,20 @@ async function acquireMcpLifecycleLock(
sandboxName,
corruptLockGraceMs,
corruptMainTracker,
monotonicNow(),
) === "stale"
) {
if (await tryReapStaleLock(lockPath, sandboxName, corruptLockGraceMs, corruptMainTracker)) {
assertBeforeDeadline();
if (
await tryReapStaleLock(
lockPath,
sandboxName,
corruptLockGraceMs,
corruptMainTracker,
monotonicNow,
assertBeforeDeadline,
)
) {
continue;
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/lib/state/mcp-lifecycle-lock-identity.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ function owner(
}

function observation(lockOwner: McpLifecycleLockOwner | null, mtimeMs = 0): LockObservation {
return { owner: lockOwner, mtimeMs, dev: 10, ino: 20 };
return { owner: lockOwner, mtimeMs, dev: 10, ino: 20, reclaimable: true };
}

function probes(
Expand Down
6 changes: 5 additions & 1 deletion src/lib/state/mcp-lifecycle-lock-identity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ export interface LockObservation {
mtimeMs: number;
dev: number;
ino: number;
/** Directories cannot be restored with the no-overwrite hard-link protocol. */
reclaimable: boolean;
}

export type McpLifecycleLockDisposition = "active" | "stale" | "wait";
Expand Down Expand Up @@ -198,7 +200,9 @@ export function classifyMcpLifecycleLock(
): McpLifecycleLockDisposition {
const { owner } = observation;
if (!owner || owner.sandboxName !== sandboxName) {
return nowMs - observation.mtimeMs >= corruptLockGraceMs ? "stale" : "wait";
return observation.reclaimable && nowMs - observation.mtimeMs >= corruptLockGraceMs
? "stale"
: "wait";
}
// The lock coordinates local CLI processes, not independent hosts or PID
// namespaces. Never use this process's PID table to reap a foreign owner;
Expand Down
51 changes: 42 additions & 9 deletions src/lib/state/mcp-lifecycle-lock-storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ export async function readMcpLifecycleLockObservation(
try {
const stat = await fs.promises.lstat(lockPath);
if (!stat.isFile() || stat.isSymbolicLink()) {
return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino };
return {
owner: null,
mtimeMs: stat.mtimeMs,
dev: stat.dev,
ino: stat.ino,
reclaimable: !stat.isDirectory(),
};
}
} catch (statError) {
if (isErrnoException(statError) && statError.code === "ENOENT") return null;
Expand All @@ -58,7 +64,13 @@ export async function readMcpLifecycleLockObservation(
try {
const stat = await handle.stat();
if (!stat.isFile()) {
return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino };
return {
owner: null,
mtimeMs: stat.mtimeMs,
dev: stat.dev,
ino: stat.ino,
reclaimable: !stat.isDirectory(),
};
}
try {
const parsed: unknown = JSON.parse(await handle.readFile("utf8"));
Expand All @@ -67,9 +79,16 @@ export async function readMcpLifecycleLockObservation(
mtimeMs: stat.mtimeMs,
dev: stat.dev,
ino: stat.ino,
reclaimable: true,
};
} catch {
return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino };
return {
owner: null,
mtimeMs: stat.mtimeMs,
dev: stat.dev,
ino: stat.ino,
reclaimable: true,
};
}
} finally {
await handle.close();
Expand Down Expand Up @@ -97,9 +116,22 @@ export async function safelyReleaseMcpLifecycleLock(
await reclaimStaleMcpLifecycleLockGeneration(lockPath, observation);
}

async function restoreClaimedMcpLifecycleLockGeneration(
targetPath: string,
quarantinePath: string,
): Promise<void> {
try {
await fs.promises.link(quarantinePath, targetPath);
await fs.promises.rm(quarantinePath, { force: true });
} catch (error) {
if (!isErrnoException(error) || error.code !== "EEXIST") throw error;
}
Comment thread
prekshivyas marked this conversation as resolved.
}

export async function reclaimStaleMcpLifecycleLockGeneration(
targetPath: string,
expected: LockObservation,
assertAfterClaim?: () => void,
): Promise<boolean> {
const quarantinePath = `${targetPath}.reclaim-${process.pid}-${crypto.randomUUID()}`;
try {
Expand All @@ -122,6 +154,12 @@ export async function reclaimStaleMcpLifecycleLockGeneration(
claimed.ino === expected.ino
: claimed?.owner?.token === expectedToken;
if (claimedExpectedGeneration) {
try {
assertAfterClaim?.();
} catch (error) {
await restoreClaimedMcpLifecycleLockGeneration(targetPath, quarantinePath);
throw error;
}
await fs.promises.rm(quarantinePath, { force: true, recursive: true });
return true;
}
Expand All @@ -131,12 +169,7 @@ export async function reclaimStaleMcpLifecycleLockGeneration(
// quarantine name. If another generation already occupies the canonical
// path, preserve the displaced owner record for diagnosis rather than ever
// deleting an owner we did not claim.
try {
await fs.promises.link(quarantinePath, targetPath);
await fs.promises.rm(quarantinePath, { force: true });
} catch (error) {
if (!isErrnoException(error) || error.code !== "EEXIST") throw error;
}
await restoreClaimedMcpLifecycleLockGeneration(targetPath, quarantinePath);
return false;
}

Expand Down
2 changes: 1 addition & 1 deletion test/helpers/mcp-lifecycle-lock-properties.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ function owner(
}

function observation(lockOwner: McpLifecycleLockOwner | null, mtimeMs = 0): LockObservation {
return { owner: lockOwner, mtimeMs, dev: 1, ino: 1 };
return { owner: lockOwner, mtimeMs, dev: 1, ino: 1, reclaimable: true };
}

function probes(
Expand Down
Loading
Loading