# Files changed · PR \#23842 · pingdotgg/t3code [View on GitCafe](https://git.cafe/pingdotgg/t3code/pull/23842/diffs) Repository: [pingdotgg/t3code](https://git.cafe/pingdotgg/t3code) Visibility: public [fix\(codex\): stalled HTTP 429 retries end with an explicit failure instead of hanging](https://git.cafe/pingdotgg/t3code/pull/23842.md) Version: 29 Head commit: 3782d8adf38639ba27e23738fa425e3cba2111d5 Base commit: 51273020db3245f4ec29514a419d29b5dcb012dc Comparison base: 51273020db3245f4ec29514a419d29b5dcb012dc Showing up to 20 entries from this revision. If the pull request changes, reopen the first page before continuing. ## apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts Status: modified; +3295; −0 ```diff @@ -40,12 +40,14 @@ import * as FileSystem from "effect/FileSystem"; import { MCP_APP_OUTPUT_KEY, readMcpAppReference } from "@t3tools/shared/mcpApp"; import { resolveAttachmentPathById } from "../../attachmentStore.ts"; +import { CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS } from "../../provider/codexRateLimitRetry.ts"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as PubSub from "effect/PubSub"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import * as Scheduler from "effect/Scheduler"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -2179,6 +2181,383 @@ ), ); + // regression: #17503 a post-tool HTTP 429 retry left the turn running for hours + it.effect.each([ + { + name: "backoff", + errorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + details: "429 Too Many Requests", + retryDelayMs: 5_000, + deadlineMs: 65_000, + resetAfterMs: null, + resetAt: null, + }, + { + name: "short-retry-after", + errorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + details: "429 Too Many Requests; Retry-After: 30", + retryDelayMs: 30_000, + deadlineMs: 90_000, + // The Retry-After already elapsed while waiting, so it is no reset time. + resetAfterMs: null, + resetAt: null, + }, + { + name: "long-retry-after", + errorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + details: '429 Too Many Requests; Retry-After: 384882 {"error":{"code":"model_cooldown"}}', + retryDelayMs: null, + deadlineMs: null, + resetAfterMs: 384_882_000, + resetAt: null, + }, + { + name: "http-date-retry-after", + errorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + details: "429 Too Many Requests; Retry-After: Thu, 22 Oct 2026 12:00:00 GMT", + retryDelayMs: null, + deadlineMs: null, + resetAfterMs: null, + // The provider's own reset date is kept, so auto-resume waits for it. + resetAt: "2026-10-22T12:00:00.000Z", + }, + { + // Codex's structured status is optional; only the text names the 429. + name: "textual-429", + errorInfo: { responseStreamDisconnected: {} }, + details: "429 Too Many Requests; Retry-After: 384882", + retryDelayMs: null, + deadlineMs: null, + resetAfterMs: 384_882_000, + resetAt: null, + }, + ] as const)( + "ends a stalled post-tool 429 retry with an explicit failure ($name)", + ({ name, errorInfo, details, retryDelayMs, deadlineMs, resetAfterMs, resetAt }) => + Effect.scoped( + Effect.gen(function* () { + const scenario = `codex-rate-limit-stall-${name}`; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const command = { + type: "commandExecution", + id: "command-before-rate-limit", + command: "pwd", + cwd: "/workspace", + processId: "42", + source: "unifiedExecStartup", + commandActions: [{ type: "unknown", command: "pwd" }], + }; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Run pwd." }), + { + type: "emit_inbound", + label: "item/started/tool", + frame: { + method: "item/started", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + startedAtMs: 1782622445000, + item: { + ...command, + status: "inProgress", + aggregatedOutput: null, + exitCode: null, + durationMs: null, + }, + }, + }, + }, + { + type: "emit_inbound", + label: "item/completed/tool", + frame: { + method: "item/completed", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + completedAtMs: 1782622445010, + item: { + ...command, + status: "completed", + aggregatedOutput: "/workspace\n", + exitCode: 0, + durationMs: 10, + }, + }, + }, + }, + // Codex then sleeps for its retry without any further notification. + { + type: "emit_inbound", + label: "error/rate-limit-retry", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: true, + error: { + message: "Reconnecting... 1/5", + additionalDetails: details, + codexErrorInfo: errorInfo, + }, + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt", frame: { id: 4, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/interrupted", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ], + }); + // Each interrupt request records the clock it was sent at, so the + // assertions read observed state instead of waiting on elapsed time. + // The request is held until released, to observe what the adapter + // published while Codex had not acknowledged the interrupt yet. + const interruptRequests: Array<{ readonly atMs: number }> = []; + const interrupted = yield* Deferred.make(); + const releaseInterrupt = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? DateTime.now.pipe( + Effect.tap((now) => + Effect.sync(() => { + interruptRequests.push({ atMs: DateTime.toEpochMillis(now) }); + }), + ), + Effect.andThen(Deferred.succeed(interrupted, undefined)), + Effect.andThen(Deferred.await(releaseInterrupt)), + ) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + const retryItems = () => + harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.retry !== undefined + ? [event.turnItem] + : [], + ); + yield* harness.until(() => retryItems().length > 0); + const retryNotice = retryItems()[0]; + assert.isDefined(retryNotice); + assert.equal(retryNotice.status, "running"); + const noticedAt = retryNotice.updatedAt; + + // The retry budget (backoff or Retry-After, plus grace) is honored + // first; TestClock wakes the deadline exactly when it is due. + if (deadlineMs !== null) { + yield* TestClock.adjust(`${deadlineMs} millis`); + // adjust returns once the woken deadline fiber and the event + // consumer it fed are idle; here that is the held interrupt. The + // native turn is interrupted before its terminal is published, so + // a queued follow-up cannot start while Codex still runs it. + assert.lengthOf(interruptRequests, 1); + assert.lengthOf( + harness.terminalEvents(), + 0, + "the terminal must wait until Codex acknowledges the interrupt", + ); + } + yield* Deferred.await(interrupted); + yield* Deferred.succeed(releaseInterrupt, undefined); + yield* harness.awaitTerminal(1); + + const expectedStopMs = DateTime.toEpochMillis(noticedAt) + (deadlineMs ?? 0); + const stoppedTurn = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated" && event.providerTurn.status === "failed", + ); + assert.isDefined(stoppedTurn, "a stalled 429 retry must end the turn"); + assert.equal( + stoppedTurn.providerTurn.completedAt === null + ? null + : DateTime.toEpochMillis(stoppedTurn.providerTurn.completedAt), + expectedStopMs, + "the turn must stop at the retry deadline, not before it", + ); + assert.deepEqual( + interruptRequests.map((request) => request.atMs), + [expectedStopMs], + "Codex's own retry sleep must be interrupted once, at the deadline", + ); + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.class, "usage_limit"); + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal(terminal.failure.retryable, true); + assert.include(terminal.failure.message, "HTTP 429"); + assert.deepEqual(terminal.retry, { attempt: 1, maxAttempts: 5, retryDelayMs }); + assert.equal( + terminal.failure.resetAt ?? null, + resetAfterMs === null + ? resetAt + : DateTime.formatIso(DateTime.add(noticedAt, { milliseconds: resetAfterMs })), + ); + assert.lengthOf(harness.terminalEvents(), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps a 429 retry that resumed into context compaction running", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-retry-compaction"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const compaction = { type: "contextCompaction", id: "compaction-after-rate-limit" }; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Summarize." }), + { + type: "emit_inbound", + label: "error/rate-limit-retry", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: true, + error: { + message: "Reconnecting... 1/5", + additionalDetails: "429 Too Many Requests", + codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + }, + }, + }, + }, + { + type: "emit_inbound", + label: "item/started/compaction", + frame: { + method: "item/started", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + startedAtMs: 1782622445000, + item: compaction, + }, + }, + }, + // Compaction outlasts the 65s first-attempt retry deadline. + { + type: "emit_inbound", + label: "item/completed/compaction", + afterMs: 70_000, + frame: { + method: "item/completed", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + completedAtMs: 1782622515000, + item: compaction, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/completed", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "completed" }), + }, + }, + }, + ], + }); + const requests: Array = []; + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + Effect.sync(() => { + requests.push(method); + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Summarize.", + }), + ); + const compactionStatuses = () => + harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "compaction" + ? [event.turnItem.status] + : [], + ); + yield* harness.until(() => compactionStatuses().includes("running")); + // Pass the first-attempt deadline (65s) while compaction is running. + yield* TestClock.adjust("70 seconds"); + yield* harness.awaitTerminal(1); + + assert.equal(harness.terminalEvents()[0]?.status, "completed"); + assert.notInclude(requests, "turn/interrupt"); + assert.deepEqual(compactionStatuses(), ["running", "completed"]); + const retryStatuses = harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.retry !== undefined + ? [event.turnItem.status] + : [], + ); + assert.deepEqual(retryStatuses, ["running", "completed"]); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect("resolves retryable app-server errors on resumed provider activity", () => Effect.scoped( Effect.gen(function* () { @@ -3876,6 +4255,2922 @@ ), ); + const rateLimitRetryNotice = (nativeThreadId: string, nativeTurnId: string) => ({ + type: "emit_inbound" as const, + label: `error/rate-limit-retry/${nativeTurnId}`, + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: true, + error: { + message: "Reconnecting... 1/5", + // Longer than the allowed retry delay, so the turn stops at once. + additionalDetails: "429 Too Many Requests; Retry-After: 384882", + codexErrorInfo: { responseStreamDisconnected: { httpStatusCode: 429 } }, + }, + }, + }, + }); + // Codex reports a turn ended, confirming the interrupt that stopped it. + const rateLimitTurnInterrupted = ( + label: string, + nativeThreadId: string, + nativeTurnId: string, + ) => ({ + type: "emit_inbound" as const, + label, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }); + const rateLimitSubagentEntries = interruptSubagentCommandTranscript.entries.slice( + 0, + interruptSubagentCommandTranscript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && entry.label === "item/started/child-command", + ), + ); + + it.effect("interrupts the whole active lineage when a root 429 retry is exhausted", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-root-lineage", + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/root", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: INTERRUPT_NATIVE_TURN, status: "interrupted" }), + }, + }, + }, + { + type: "emit_inbound", + label: "turn/completed/child", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_CHILD_NATIVE_TURN, + status: "interrupted", + }), + }, + }, + }, + ], + }); + const interrupts: Array = []; + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + Effect.sync(() => { + if (method === "turn/interrupt") interrupts.push(params); + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-codex-rate-limit-root-lineage"), + text: INTERRUPT_PROMPT, + }), + ); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.isUndefined( + terminal.childTurnsContinue, + "no descendant may keep the run active past the retry deadline", + ); + assert.deepEqual(interrupts, [ + { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + { threadId: INTERRUPT_CHILD_NATIVE_THREAD, turnId: INTERRUPT_CHILD_NATIVE_TURN }, + ]); + assert.equal(harness.subagentUpdates().at(-1)?.subagent.status, "interrupted"); + assertChildProviderTerminalBeforeRoot(harness.events, harness.threadId); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("interrupts a nested turn that starts while the 429 lineage is being stopped", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-late-descendant", + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + // The subagent starts a nested turn while the root interrupt is in flight. + { + type: "emit_inbound", + label: "turn/started/late-child", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + rateLimitTurnInterrupted( + "turn/completed/child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_CHILD_NATIVE_TURN, + ), + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 6, result: {} }, + }, + rateLimitTurnInterrupted( + "turn/completed/late-child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + rateLimitTurnInterrupted( + "turn/completed/root", + INTERRUPT_NATIVE_THREAD, + INTERRUPT_NATIVE_TURN, + ), + ], + }); + const interrupts: Array = []; + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + Effect.sync(() => { + if (method === "turn/interrupt") interrupts.push(params); + }), + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-codex-rate-limit-late-descendant"), + text: INTERRUPT_PROMPT, + }), + ); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.isUndefined( + terminal.childTurnsContinue, + "a nested turn started mid-teardown may not keep the run active", + ); + assert.deepEqual(interrupts, [ + { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + { threadId: INTERRUPT_CHILD_NATIVE_THREAD, turnId: INTERRUPT_CHILD_NATIVE_TURN }, + { threadId: INTERRUPT_CHILD_NATIVE_THREAD, turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN }, + ]); + const lateChildUpdates = harness.events.filter( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + const lateChildSettled = lateChildUpdates.at(-1); + assert.isDefined(lateChildSettled); + assert.equal(lateChildSettled.providerTurn.status, "interrupted"); + assert.isBelow( + harness.events.indexOf(lateChildSettled), + harness.events.indexOf(terminal), + "the nested turn must settle before the root fails", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect( + "settles a nested turn started during the 429 root interrupt no earlier than it started", + () => + Effect.scoped( + Effect.gen(function* () { + // The deadline stops the root at DEADLINE_AT; the nested turn starts + // two seconds later, while the root interrupt is still in flight. + const DEADLINE_AT_MS = 1782622445000; + const LATE_CHILD_STARTED_AT_SECONDS = DEADLINE_AT_MS / 1000 + 2; + yield* TestClock.setTime(DEADLINE_AT_MS); + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-late-descendant-completed-at", + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + // The subagent starts a nested turn while the root interrupt is in flight. + { + type: "emit_inbound", + label: "turn/started/late-child", + afterMs: 5_000, + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: { + ...makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + // Codex starts it after the deadline stopped the root. + startedAt: LATE_CHILD_STARTED_AT_SECONDS, + }, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + rateLimitTurnInterrupted( + "turn/completed/child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_CHILD_NATIVE_TURN, + ), + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 6, result: {} }, + }, + rateLimitTurnInterrupted( + "turn/completed/late-child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + rateLimitTurnInterrupted( + "turn/completed/root", + INTERRUPT_NATIVE_THREAD, + INTERRUPT_NATIVE_TURN, + ), + ], + }); + const rootInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(rootInterruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-codex-rate-limit-late-descendant-completed-at"), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(rootInterruptSent); + yield* TestClock.adjust("5 seconds"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + const lateChildSettled = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.isDefined(lateChildSettled); + assert.equal(lateChildSettled.providerTurn.status, "interrupted"); + const { startedAt, completedAt } = lateChildSettled.providerTurn; + assert.isNotNull(startedAt); + assert.isNotNull(completedAt); + if (startedAt === null || completedAt === null) return; + assert.equal(DateTime.toEpochMillis(startedAt), LATE_CHILD_STARTED_AT_SECONDS * 1000); + assert.isAtLeast( + DateTime.toEpochMillis(completedAt), + DateTime.toEpochMillis(startedAt), + "a nested turn cannot complete before it started", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("interrupts a nested turn that starts after the last 429 lineage scan", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-final-scan-descendant", + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + rateLimitTurnInterrupted( + "turn/completed/child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_CHILD_NATIVE_TURN, + ), + rateLimitTurnInterrupted( + "turn/completed/root", + INTERRUPT_NATIVE_THREAD, + INTERRUPT_NATIVE_TURN, + ), + // The subagent starts a nested turn right after Codex confirms the + // lineage stopped, as the teardown makes its final lineage scan. + { + type: "emit_inbound", + label: "turn/started/late-child", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 6, result: {} }, + }, + rateLimitTurnInterrupted( + "turn/completed/late-child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + ], + }); + const lateChildInterrupted = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + method === "turn/interrupt" && + (params as { readonly turnId?: string }).turnId === INTERRUPT_LATE_CHILD_NATIVE_TURN + ? Deferred.succeed(lateChildInterrupted, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-codex-rate-limit-final-scan-descendant"), + text: INTERRUPT_PROMPT, + }), + ); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.isUndefined( + terminal.childTurnsContinue, + "a nested turn started during the final scan may not keep the run active", + ); + // Either way the late nested turn is stopped rather than left running. + yield* Deferred.await(lateChildInterrupted); + }).pipe( + // A small yield budget interleaves the late turn/started handler with + // the teardown's final lineage scan, the window this test covers. + Effect.provideService(Scheduler.MaxOpsBeforeYield, 8), + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps an exhausted 429 turn active until Codex confirms it stopped", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-uncontained"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Run pwd." }), + rateLimitRetryNotice(nativeThreadId, nativeTurnId), + // Codex never answers the deadline's interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/deadline", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/stop", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/stop", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/interrupted", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ], + }); + const deadlineInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(deadlineInterruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(deadlineInterruptSent); + // adjust returns once the interrupt timeout fired and its fiber is idle. + yield* TestClock.adjust("10 seconds"); + const failedTurns = () => + harness.events.filter( + (event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "failed", + ); + assert.lengthOf( + harness.terminalEvents(), + 0, + "an unacknowledged interrupt must not make the thread idle", + ); + assert.lengthOf(failedTurns(), 0); + + const providerTurnId = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + // The turn is still stoppable, and settles once Codex ends it. + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + yield* harness.awaitTerminal(1); + + assert.lengthOf(failedTurns(), 1); + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.lengthOf(harness.terminalEvents(), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + // Codex never answers the deadline's turn/interrupt, so the turn is + // "unconfirmed"; `lateEntries` follow that interrupt. + const unconfirmedRateLimitTranscript = ( + scenario: string, + lateEntries: ReadonlyArray = [], + ) => { + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + return makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Run pwd." }), + rateLimitRetryNotice(nativeThreadId, nativeTurnId), + { + type: "expect_outbound", + label: "turn/interrupt/deadline", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + ...lateEntries, + ], + }); + }; + const failedProviderTurns = (events: ReadonlyArray) => + events.filter( + (event) => event.type === "provider_turn.updated" && event.providerTurn.status === "failed", + ); + + it.effect("fails an unconfirmed 429 turn as broken once Codex stays silent past the bound", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-bound"; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + // The interrupt times out, so the turn is kept active as unconfirmed. + yield* TestClock.adjust("10 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS - 1} millis`); + assert.lengthOf(harness.terminalEvents(), 0, "the turn waits for the whole bound"); + yield* TestClock.adjust("1 millis"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal(terminal.failure.class, "usage_limit"); + assert.isTrue(terminal.failure.retryable); + assert.equal( + terminal.threadDisposition, + "broken", + "a turn Codex never confirmed stopped must not reuse the loaded thread", + ); + assert.lengthOf(failedProviderTurns(harness.events), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("waits for turn/completed after a 429 interrupt reply before ending the turn", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-interrupt-reply-only"; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + // Codex accepts the interrupt but never reports the turn ended. + unconfirmedRateLimitTranscript(scenario, [ + { + type: "emit_inbound", + label: "turn/interrupt/deadline", + frame: { id: 4, result: {} }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + // The reply alone does not confirm the stop, so the turn waits for + // turn/completed within the interrupt bound, then the unconfirmed + // bound. Stepping lets the reply be handled before the bound ends. + for (let second = 0; second < 10; second++) { + yield* TestClock.adjust("1 seconds"); + } + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal( + terminal.threadDisposition, + "broken", + "a turn Codex never reported ended must not reuse the loaded thread", + ); + assert.lengthOf(failedProviderTurns(harness.events), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("ends a 429 turn once turn/completed follows its interrupt reply", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-interrupt-reply-then-completed"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + { + type: "emit_inbound", + label: "turn/interrupt/deadline", + frame: { id: 4, result: {} }, + }, + { + type: "emit_inbound", + label: "turn/completed/interrupted", + afterMs: 5_000, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + const interruptAt = DateTime.toEpochMillis(yield* DateTime.now); + yield* TestClock.adjust("5 seconds"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.notEqual( + terminal.threadDisposition, + "broken", + "Codex confirmed the stop, so the loaded thread stays reusable", + ); + const rootFailed = failedProviderTurns(harness.events)[0]; + if (rootFailed?.type !== "provider_turn.updated") return assert.fail("no failed turn"); + const completedAt = rootFailed.providerTurn.completedAt; + assert.isNotNull(completedAt); + if (completedAt === null) return; + // The turn still settles at the deadline that stopped it. + assert.equal(DateTime.toEpochMillis(completedAt), interruptAt); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect( + "marks an unconfirmed 429 root thread errored so its session drops the loaded thread", + () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-root-thread-error"; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + yield* TestClock.adjust("10 seconds"); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + + const terminalIndex = harness.events.findIndex((event) => event.type === "turn.terminal"); + const rootThreadErrorIndex = harness.events.findIndex( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.id === harness.providerThread.id && + event.providerThread.appThreadId === harness.threadId && + event.providerThread.status === "error", + ); + // The session's loaded-thread cache only drops a thread on an errored + // provider_thread.updated; the broken disposition alone would let the + // next turn start on the native thread that may still be retrying. + assert.notEqual(rootThreadErrorIndex, -1, "the root thread must be published as errored"); + assert.isBelow(rootThreadErrorIndex, terminalIndex); + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.threadDisposition, "broken"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("ignores a turn/completed that arrives after the unconfirmed 429 bound", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-late-completion"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + // 10s interrupt timeout + the bound, then 5s more. + const lateCompletionMs = 10_000 + CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS + 5_000; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + { + type: "emit_inbound", + label: "turn/completed/late", + afterMs: lateCompletionMs, + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + yield* TestClock.adjust("10 seconds"); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.threadDisposition, "broken"); + + // adjust returns once the late turn/completed has been handled. + yield* TestClock.adjust("5 seconds"); + assert.lengthOf(harness.terminalEvents(), 1, "a late turn/completed must not end it twice"); + assert.lengthOf(failedProviderTurns(harness.events), 1); + assert.notInclude( + harness.events.map((event) => + event.type === "provider_turn.updated" ? event.providerTurn.status : null, + ), + "interrupted", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps an exhausted 429 turn active when its interrupt request fails", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-interrupt-error"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + { + type: "emit_inbound", + label: "turn/interrupt/deadline", + afterMs: 1_000, + frame: { id: 4, error: { code: -32_603, message: "Interrupt failed" } }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/stop", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/stop", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/interrupted", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + // adjust returns once the interrupt's error response has been handled. + yield* TestClock.adjust("1 second"); + assert.lengthOf( + harness.terminalEvents(), + 0, + "a failed interrupt must not make the thread idle", + ); + assert.lengthOf(failedProviderTurns(harness.events), 0); + + const providerTurnId = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal( + terminal.threadDisposition, + "reusable", + "Codex confirmed the turn ended, so its thread stays reusable", + ); + + // The confirmed stop cancelled the fallback: passing its bound ends nothing. + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + assert.lengthOf(harness.terminalEvents(), 1); + assert.lengthOf(failedProviderTurns(harness.events), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("stops a nested turn that starts during the unconfirmed 429 wait", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-late-descendant"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + // Codex never answers the root's deadline interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + // The subagent starts a nested turn while the root waits, unconfirmed. + { + type: "emit_inbound", + label: "turn/started/late-child", + afterMs: 20_000, + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 6, result: {} }, + }, + ], + }); + const interrupts: Array = []; + const rootInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + method === "turn/interrupt" + ? Effect.sync(() => interrupts.push(params)).pipe( + Effect.andThen(Deferred.succeed(rootInterruptSent, undefined)), + Effect.asVoid, + ) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(rootInterruptSent); + // The root interrupt times out; the existing child is stopped. + yield* TestClock.adjust("10 seconds"); + // The late nested turn starts inside the unconfirmed wait. + yield* TestClock.adjust("20 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + assert.isTrue( + harness.events.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + "the nested turn must have started before the bound", + ); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal(terminal.threadDisposition, "broken"); + assert.isUndefined( + terminal.childTurnsContinue, + "a nested turn started during the unconfirmed wait may not keep the run active", + ); + assert.deepInclude(interrupts, { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }); + const lateChildSettled = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.isDefined(lateChildSettled); + assert.equal(lateChildSettled.providerTurn.status, "interrupted"); + assert.isBelow( + harness.events.indexOf(lateChildSettled), + harness.events.indexOf(terminal), + "the nested turn must settle before the root fails", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect( + "stops a nested turn that started during the unconfirmed 429 wait when Codex ends the root", + () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-completed-late-descendant"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + // Codex never answers the root's deadline interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + rateLimitTurnInterrupted( + "turn/completed/child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_CHILD_NATIVE_TURN, + ), + // The subagent starts a nested turn while the root waits, unconfirmed. + { + type: "emit_inbound", + label: "turn/started/late-child", + afterMs: 20_000, + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + // Codex then confirms the root ended, well inside the unconfirmed bound. + { + type: "emit_inbound", + label: "turn/completed/root", + afterMs: 5_000, + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: INTERRUPT_NATIVE_TURN, status: "interrupted" }), + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 6, result: {} }, + }, + rateLimitTurnInterrupted( + "turn/completed/late-child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + ], + }); + const interrupts: Array = []; + const rootInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + method === "turn/interrupt" + ? Effect.sync(() => interrupts.push(params)).pipe( + Effect.andThen(Deferred.succeed(rootInterruptSent, undefined)), + Effect.asVoid, + ) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(rootInterruptSent); + // The root interrupt times out; the existing child is stopped. + yield* TestClock.adjust("10 seconds"); + // The late nested turn starts inside the unconfirmed wait. + yield* TestClock.adjust("20 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + assert.isTrue( + harness.events.some( + (event) => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ), + "the nested turn must have started before the root ended", + ); + yield* TestClock.adjust("5 seconds"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.isUndefined( + terminal.childTurnsContinue, + "a nested turn started during the unconfirmed wait may not outlive the root", + ); + assert.deepInclude(interrupts, { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }); + const lateChildSettled = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.isDefined(lateChildSettled); + assert.equal(lateChildSettled.providerTurn.status, "interrupted"); + assert.isBelow( + harness.events.indexOf(lateChildSettled), + harness.events.indexOf(terminal), + "the nested turn must settle before the root fails", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + const lastChildThreadStatus = (events: ReadonlyArray) => + events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_thread.updated" } + > => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === INTERRUPT_CHILD_NATIVE_THREAD, + )?.providerThread.status; + + it.effect("starts the unconfirmed 429 bound one interrupt bound after silent descendants", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-silent-descendants"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + // A second nested turn is active when the lineage is stopped. + { + type: "emit_inbound", + label: "turn/started/late-child", + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + // Codex accepts the root's interrupt but never reports it ended. + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + // Codex never answers either nested turn's interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + ], + }); + const rootInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(rootInterruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(rootInterruptSent); + const rootInterruptAt = DateTime.toEpochMillis(yield* DateTime.now); + for (let second = 0; second < 120 && harness.terminalEvents().length === 0; second++) { + yield* TestClock.adjust("1 seconds"); + } + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + assert.equal(terminal?.threadDisposition, "broken"); + const rootFailed = failedProviderTurns(harness.events)[0]; + if (rootFailed?.type !== "provider_turn.updated") return assert.fail("root not failed"); + const rootCompletedAt = rootFailed.providerTurn.completedAt; + assert.isNotNull(rootCompletedAt); + if (rootCompletedAt === null) return; + // Silent descendants share one 10s interrupt bound instead of adding + // 10s each; the clock steps 1s at a time, so allow one step. + const settledAfterMs = DateTime.toEpochMillis(rootCompletedAt) - rootInterruptAt; + assert.isAtLeast(settledAfterMs, 10_000 + CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS); + assert.isAtMost( + settledAfterMs, + 10_000 + CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS + 1_000, + "the unconfirmed bound must not wait on each silent descendant in turn", + ); + assert.equal(lastChildThreadStatus(harness.events), "error"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("settles a 429 lineage at the unconfirmed bound when a nested turn stays silent", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-silent-late-descendant"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + // Codex never answers the root's deadline interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + rateLimitTurnInterrupted( + "turn/completed/child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_CHILD_NATIVE_TURN, + ), + // The subagent starts a nested turn while the root waits, unconfirmed. + { + type: "emit_inbound", + label: "turn/started/late-child", + afterMs: 20_000, + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + // Codex never answers the nested turn's interrupt either. + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + ], + }); + const interrupts: Array = []; + const rootInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + method === "turn/interrupt" + ? Effect.sync(() => interrupts.push(params)).pipe( + Effect.andThen(Deferred.succeed(rootInterruptSent, undefined)), + Effect.asVoid, + ) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(rootInterruptSent); + const rootInterruptAt = DateTime.toEpochMillis(yield* DateTime.now); + // The root interrupt times out and the unconfirmed bound starts. + yield* TestClock.adjust("10 seconds"); + const boundAt = rootInterruptAt + 10_000 + CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS; + // The late nested turn starts inside the unconfirmed wait. + yield* TestClock.adjust("20 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS - 20_000} millis`); + // Let a lineage cleanup that waits on the silent nested turn finish too, so a + // late settle is measured instead of hanging the test. + yield* TestClock.adjust("10 seconds"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal(terminal.threadDisposition, "broken"); + const rootFailed = failedProviderTurns(harness.events)[0]; + assert.isDefined(rootFailed); + if (rootFailed?.type !== "provider_turn.updated") return; + const rootCompletedAt = rootFailed.providerTurn.completedAt; + assert.isNotNull(rootCompletedAt); + if (rootCompletedAt === null) return; + // The unconfirmed bound caps the whole lineage, not just the root. + assert.equal( + DateTime.toEpochMillis(rootCompletedAt) - boundAt, + 0, + "the root must settle at the unconfirmed bound", + ); + assert.deepInclude(interrupts, { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }); + const lateChildSettled = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.isDefined(lateChildSettled); + assert.equal(lateChildSettled.providerTurn.status, "interrupted"); + assert.isBelow( + harness.events.indexOf(lateChildSettled), + harness.events.indexOf(terminal), + "the nested turn must settle before the root fails", + ); + // Its stop was never confirmed, so its thread may still be running. + assert.equal(lastChildThreadStatus(harness.events), "error"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("marks a descendant errored when the 429 deadline cannot interrupt it", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-descendant-interrupt-error", + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + // Codex rejects the child's interrupt, so it may still be running. + { + type: "emit_inbound", + label: "turn/interrupt/child", + frame: { id: 5, error: { code: -32_603, message: "Interrupt failed" } }, + }, + rateLimitTurnInterrupted( + "turn/completed/root", + INTERRUPT_NATIVE_THREAD, + INTERRUPT_NATIVE_TURN, + ), + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-codex-rate-limit-descendant-interrupt-error"), + text: INTERRUPT_PROMPT, + }), + ); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + assert.equal( + lastChildThreadStatus(harness.events), + "error", + "a descendant Codex never confirmed stopped must not be published as idle", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("marks a descendant errored when Codex only replies to its 429 interrupt", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-descendant-interrupt-reply-only"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 4, result: {} } }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + // Codex accepts the child's interrupt but never reports it ended. + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/root", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_NATIVE_THREAD, + turn: makeCodexReplayTurn({ id: INTERRUPT_NATIVE_TURN, status: "interrupted" }), + }, + }, + }, + ], + }); + const childInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method, params) => + method === "turn/interrupt" && + (params as { readonly threadId?: unknown } | undefined)?.threadId === + INTERRUPT_CHILD_NATIVE_THREAD + ? Deferred.succeed(childInterruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(childInterruptSent); + yield* TestClock.adjust("10 seconds"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + assert.equal( + lastChildThreadStatus(harness.events), + "error", + "a descendant Codex never reported ended must not be published as idle", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("marks a late descendant errored when the unconfirmed 429 bound cannot stop it", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-late-descendant-error"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + // Codex never answers the root's deadline interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 5, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 5, result: {} } }, + { + type: "emit_inbound", + label: "turn/started/late-child", + afterMs: 20_000, + frame: { + method: "turn/started", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_LATE_CHILD_NATIVE_TURN, + status: "inProgress", + }), + }, + }, + }, + { + type: "expect_outbound", + label: "turn/interrupt/late-child", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + // Codex rejects the late nested turn's interrupt as well. + { + type: "emit_inbound", + label: "turn/interrupt/late-child", + frame: { id: 6, error: { code: -32_603, message: "Interrupt failed" } }, + }, + ], + }); + const rootInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(rootInterruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(rootInterruptSent); + yield* TestClock.adjust("10 seconds"); + yield* TestClock.adjust("20 seconds"); + assert.equal( + lastChildThreadStatus(harness.events), + "active", + "the nested turn must have started on the child thread before the bound", + ); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.threadDisposition, "broken"); + const lateChildSettled = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.equal(lateChildSettled?.providerTurn.status, "interrupted"); + assert.equal( + lastChildThreadStatus(harness.events), + "error", + "a nested turn Codex never confirmed stopped must not be published as idle", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("marks a late descendant errored when Codex only replies to Stop of its 429 turn", () => + Effect.scoped( + Effect.gen(function* () { + const lateChildStartedIndex = interruptLateSubagentTurnTranscript.entries.findIndex( + (entry) => entry.type === "emit_inbound" && entry.label === "turn/started/late-child", + ); + const lateChildStopIndex = interruptLateSubagentTurnTranscript.entries.findIndex( + (entry) => + entry.type === "expect_outbound" && entry.label === "turn/interrupt/late-child", + ); + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-late-descendant-stop-reply-only", + entries: [ + ...interruptLateSubagentTurnTranscript.entries.slice(0, lateChildStartedIndex + 1), + // The late nested turn hits a 429 while Stop waits, and Codex + // rejects the deadline's interrupt, so its stop is unconfirmed. + rateLimitRetryNotice(INTERRUPT_CHILD_NATIVE_THREAD, INTERRUPT_LATE_CHILD_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/late-child-deadline", + frame: { + id: 7, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child-deadline", + frame: { id: 7, error: { code: -32603, message: "interrupt failed" } }, + }, + ...interruptLateSubagentTurnTranscript.entries.slice( + lateChildStartedIndex + 1, + lateChildStopIndex, + ), + // Stop's late-descendant pass then interrupts it. Codex replies + // but never reports the nested turn ended. + { + type: "expect_outbound", + label: "turn/interrupt/late-child-stop", + frame: { + id: 8, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_LATE_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/late-child-stop", + frame: { id: 8, result: {} }, + }, + { type: "runtime_exit", status: "success" }, + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + const now = yield* DateTime.now; + + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now, + attemptId: RunAttemptId.make( + "attempt-codex-rate-limit-late-descendant-stop-reply-only", + ), + text: INTERRUPT_PROMPT, + }), + ); + yield* harness.until(() => + harness.events.some( + (event) => + event.type === "turn_item.updated" && + event.turnItem.type === "command_execution" && + event.turnItem.nativeItemRef?.nativeId === INTERRUPT_CHILD_COMMAND_ITEM && + event.turnItem.status === "running", + ), + ); + const providerTurnId = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated" && event.threadId === harness.threadId, + )?.providerTurn.id; + assert.isDefined(providerTurnId); + + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + yield* harness.awaitTerminal(1); + + assert.equal(harness.terminalEvents()[0]?.status, "interrupted"); + const lateChildUpdates = harness.events.filter( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => + event.type === "provider_turn.updated" && + event.providerTurn.nativeTurnRef?.nativeId === INTERRUPT_LATE_CHILD_NATIVE_TURN, + ); + assert.equal(lateChildUpdates.at(-1)?.providerTurn.status, "interrupted"); + assert.equal( + lastChildThreadStatus(harness.events), + "error", + "a reply to Stop does not confirm the nested turn ended, so its thread must not be idle", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("marks an abandoned subagent 429 thread as errored rather than idle", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-subagent-abandoned"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_CHILD_NATIVE_THREAD, INTERRUPT_CHILD_NATIVE_TURN), + // Codex never answers the subagent's deadline interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 4, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + ], + }); + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(interruptSent); + yield* TestClock.adjust("10 seconds"); + assert.notEqual(harness.subagentUpdates().at(-1)?.subagent.status, "failed"); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.until(() => harness.subagentUpdates().at(-1)?.subagent.status === "failed"); + + const childThreadUpdate = harness.events.findLast( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_thread.updated" } + > => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === INTERRUPT_CHILD_NATIVE_THREAD, + ); + assert.isDefined(childThreadUpdate); + assert.equal( + childThreadUpdate.providerThread.status, + "error", + "a subagent thread Codex never confirmed stopped must not be reused as idle", + ); + assert.lengthOf(harness.terminalEvents(), 0, "the root turn keeps running"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps an unconfirmed subagent 429 thread idle once the root deadline stops it", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-descendant-stopped-by-root"; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_CHILD_NATIVE_THREAD, INTERRUPT_CHILD_NATIVE_TURN), + // Codex never answers the subagent's own deadline interrupt. + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 4, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + // While the subagent waits unconfirmed, the root's retry stalls too. + { + ...rateLimitRetryNotice(INTERRUPT_NATIVE_THREAD, INTERRUPT_NATIVE_TURN), + afterMs: 20_000, + }, + { + type: "expect_outbound", + label: "turn/interrupt/root", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: INTERRUPT_NATIVE_THREAD, turnId: INTERRUPT_NATIVE_TURN }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/root", frame: { id: 5, result: {} } }, + // The root's lineage stop interrupts the subagent, and Codex confirms it. + { + type: "expect_outbound", + label: "turn/interrupt/child-again", + frame: { + id: 6, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/child-again", + frame: { id: 6, result: {} }, + }, + rateLimitTurnInterrupted( + "turn/completed/child", + INTERRUPT_CHILD_NATIVE_THREAD, + INTERRUPT_CHILD_NATIVE_TURN, + ), + rateLimitTurnInterrupted( + "turn/completed/root", + INTERRUPT_NATIVE_THREAD, + INTERRUPT_NATIVE_TURN, + ), + ], + }); + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + transcript, + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: INTERRUPT_PROMPT, + }), + ); + yield* Deferred.await(interruptSent); + // The subagent's interrupt times out, leaving its stop unconfirmed. + yield* TestClock.adjust("10 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + // The root's 429 notice arrives inside the subagent's unconfirmed bound. + yield* TestClock.adjust("10 seconds"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.isUndefined(terminal.childTurnsContinue); + assert.equal( + lastChildThreadStatus(harness.events), + "idle", + "a subagent whose interrupt Codex confirmed is stopped, not broken", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps an unconfirmed 429 thread broken when Stop cannot interrupt it", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-stop-error"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + { + type: "expect_outbound", + label: "turn/interrupt/stop", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { + type: "emit_inbound", + label: "turn/interrupt/stop", + frame: { id: 5, error: { code: -32_603, message: "Interrupt failed" } }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + // The deadline's interrupt times out, so the turn waits unconfirmed. + yield* TestClock.adjust("10 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + + const providerTurnId = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + // Stop arrives before the bound, and Codex rejects its interrupt too. + const stopped = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.result); + assert.equal(stopped._tag, "Failure"); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.isDefined(terminal); + assert.equal( + terminal.threadDisposition, + "broken", + "Codex confirmed neither stop, so the loaded thread must not be reused", + ); + + // Stop owns the terminal: passing the bound ends nothing again. + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + assert.lengthOf(harness.terminalEvents(), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps an unconfirmed 429 thread broken when Codex only replies to Stop", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-stop-reply-only"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const interruptSent = yield* Deferred.make(); + const stopInterruptSent = yield* Deferred.make(); + let interruptsSent = 0; + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + { + type: "expect_outbound", + label: "turn/interrupt/stop", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + // Codex accepts Stop's interrupt but never reports the turn ended. + { type: "emit_inbound", label: "turn/interrupt/stop", frame: { id: 5, result: {} } }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed( + ++interruptsSent === 1 ? interruptSent : stopInterruptSent, + undefined, + ).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + // The deadline's interrupt times out, so the turn waits unconfirmed. + yield* TestClock.adjust("10 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + + const providerTurnId = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + // Stop arrives before the bound. Codex replies to its interrupt, so + // Stop waits for turn/completed until its 10s interrupt timeout. + const stopFiber = yield* harness.runtime + .interruptTurn({ providerThread: harness.providerThread, providerTurnId }) + .pipe(Effect.forkScoped); + yield* Deferred.await(stopInterruptSent); + for (let second = 0; second < 9; second++) { + yield* TestClock.adjust("1 seconds"); + } + assert.isUndefined(stopFiber.pollUnsafe(), "Stop waits for turn/completed"); + // Stepping lets the reply be handled before the timeout is counted. + for (let second = 0; second < 3 && stopFiber.pollUnsafe() === undefined; second++) { + yield* TestClock.adjust("1 seconds"); + } + yield* Fiber.join(stopFiber); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "interrupted"); + assert.equal( + terminal?.threadDisposition, + "broken", + "a reply to Stop does not confirm the turn ended, so the loaded thread must not be reused", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("reuses an unconfirmed 429 thread once turn/completed follows Stop", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-stop-completed"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + { + type: "expect_outbound", + label: "turn/interrupt/stop", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/stop", frame: { id: 5, result: {} } }, + rateLimitTurnInterrupted("turn/completed/stop", nativeThreadId, nativeTurnId), + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + // The deadline's interrupt times out, so the turn waits unconfirmed. + yield* TestClock.adjust("10 seconds"); + assert.lengthOf(harness.terminalEvents(), 0); + + const providerTurnId = harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated", + )?.providerTurn.id; + assert.isDefined(providerTurnId); + // Codex reports the turn ended after Stop's interrupt. + yield* harness.runtime.interruptTurn({ + providerThread: harness.providerThread, + providerTurnId, + }); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.isDefined(terminal); + assert.equal( + terminal.threadDisposition, + "reusable", + "turn/completed confirmed the stop, so the loaded thread stays reusable", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect( + "keeps a 429 thread broken when Stop ends it before the deadline's stop is confirmed", + () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-stop-before-deadline-confirmed"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const stopInterruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Run pwd." }), + { + type: "expect_outbound", + label: "turn/interrupt/stop", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + // Codex accepts Stop's interrupt but never reports the turn ended. + { + type: "emit_inbound", + label: "turn/interrupt/stop", + frame: { id: 4, result: {} }, + }, + // The retry deadline fires while Stop still waits for turn/completed. + { ...rateLimitRetryNotice(nativeThreadId, nativeTurnId), afterMs: 3_000 }, + // Codex never answers the deadline's interrupt either. + { + type: "expect_outbound", + label: "turn/interrupt/deadline", + frame: { + id: 5, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + ], + }), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(stopInterruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + const providerTurnId = () => + harness.events.find( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_turn.updated" } + > => event.type === "provider_turn.updated", + )?.providerTurn.id; + yield* harness.until(() => providerTurnId() !== undefined); + const stopFiber = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: providerTurnId()!, + }) + .pipe(Effect.forkScoped); + yield* Deferred.await(stopInterruptSent); + // Stop's 10s wait ends before the deadline's own interrupt wait does. + for (let second = 0; second < 15 && stopFiber.pollUnsafe() === undefined; second++) { + yield* TestClock.adjust("1 seconds"); + } + yield* Fiber.join(stopFiber); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "interrupted"); + assert.equal( + terminal?.threadDisposition, + "broken", + "neither interrupt was confirmed by turn/completed, so the loaded thread must not be reused", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps the 429 retry metadata when compaction starts after the deadline", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-compaction"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + // Compaction starts while the stopped turn waits, unconfirmed. + { + type: "emit_inbound", + label: "item/started/compaction", + afterMs: 12_000, + frame: { + method: "item/started", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + startedAtMs: 1782622445000, + item: { type: "contextCompaction", id: `${scenario}-compaction` }, + }, + }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + const compactionStarted = () => + harness.events.some( + (event) => event.type === "turn_item.updated" && event.turnItem.type === "compaction", + ); + for (let second = 0; second < 15 && !compactionStarted(); second++) { + yield* TestClock.adjust("1 seconds"); + } + assert.isTrue(compactionStarted()); + assert.lengthOf(harness.terminalEvents(), 0); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.deepEqual( + terminal.retry, + { attempt: 1, maxAttempts: 5, retryDelayMs: null }, + "the exhausted retry keeps its metadata on the failure", + ); + const retryStatuses = harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.retry !== undefined + ? [event.turnItem.status] + : [], + ); + assert.notInclude(retryStatuses, "completed", "an exhausted retry never completes"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps the latest goal when an unconfirmed 429 marks the thread errored", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-unconfirmed-goal"; + const nativeThreadId = `native-${scenario}-thread`; + const interruptSent = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness( + unconfirmedRateLimitTranscript(scenario, [ + // Codex updates the thread's goal while the stopped turn waits. + { + type: "emit_inbound", + label: "thread/goal/updated", + afterMs: 12_000, + frame: { + method: "thread/goal/updated", + params: { + threadId: nativeThreadId, + goal: { + threadId: nativeThreadId, + objective: "Run pwd.", + status: "complete", + tokenBudget: null, + tokensUsed: 1200, + timeUsedSeconds: 3, + createdAt: 1782622440, + updatedAt: 1782622450, + }, + }, + }, + }, + ]), + () => Effect.void, + (method) => + method === "turn/interrupt" + ? Deferred.succeed(interruptSent, undefined).pipe(Effect.asVoid) + : Effect.void, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* Deferred.await(interruptSent); + const rootThreadUpdates = () => + harness.events.filter( + ( + event, + ): event is Extract< + ProviderAdapter.ProviderAdapterV2Event, + { type: "provider_thread.updated" } + > => + event.type === "provider_thread.updated" && + event.providerThread.nativeThreadRef?.nativeId === nativeThreadId, + ); + for (let second = 0; second < 15 && rootThreadUpdates().length === 0; second++) { + yield* TestClock.adjust("1 seconds"); + } + const goalUpdate = rootThreadUpdates()[0]; + assert.isDefined(goalUpdate); + assert.equal(goalUpdate?.providerThread.goal?.status, "complete"); + yield* TestClock.adjust(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + yield* harness.awaitTerminal(1); + + const errored = rootThreadUpdates().findLast( + (event) => event.providerThread.status === "error", + ); + assert.isDefined(errored); + assert.deepEqual( + errored?.providerThread.goal, + goalUpdate?.providerThread.goal, + "marking the thread errored must not rewind its goal", + ); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("keeps the 429 failure when a final error arrives during the interrupt", () => + Effect.scoped( + Effect.gen(function* () { + const scenario = "codex-rate-limit-late-error"; + const nativeThreadId = `native-${scenario}-thread`; + const nativeTurnId = `native-${scenario}-turn`; + const transcript = makeCodexReplayTranscript({ + scenario, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt: "Run pwd." }), + rateLimitRetryNotice(nativeThreadId, nativeTurnId), + { + type: "expect_outbound", + label: "turn/interrupt", + frame: { + id: 4, + method: "turn/interrupt", + params: { threadId: nativeThreadId, turnId: nativeTurnId }, + }, + }, + // Codex gives up on its own retry while the interrupt is pending. + { + type: "emit_inbound", + label: "error/final", + frame: { + method: "error", + params: { + threadId: nativeThreadId, + turnId: nativeTurnId, + willRetry: false, + error: { + message: "exceeded retry limit, last status: 500 Internal Server Error", + codexErrorInfo: { responseTooManyFailedAttempts: { httpStatusCode: 500 } }, + }, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt", frame: { id: 4, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/interrupted", + frame: { + method: "turn/completed", + params: { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "interrupted" }), + }, + }, + }, + ], + }); + const harness = yield* makeCodexReplayHarness(transcript, () => Effect.void); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make(`attempt-${scenario}`), + text: "Run pwd.", + }), + ); + yield* harness.awaitTerminal(1); + + const terminal = harness.terminalEvents()[0]; + assert.equal(terminal?.status, "failed"); + if (terminal?.status !== "failed") return; + assert.equal(terminal.failure.code, "rateLimitRetryExhausted"); + assert.equal(terminal.failure.class, "usage_limit"); + assert.isDefined(terminal.failure.resetAt, "the Retry-After reset time must survive"); + assert.lengthOf(harness.terminalEvents(), 1); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + + it.effect("fails an exhausted subagent 429 retry with the usage-limit failure", () => + Effect.scoped( + Effect.gen(function* () { + const transcript = makeCodexReplayTranscript({ + scenario: "codex-rate-limit-subagent", + entries: [ + ...rateLimitSubagentEntries, + rateLimitRetryNotice(INTERRUPT_CHILD_NATIVE_THREAD, INTERRUPT_CHILD_NATIVE_TURN), + { + type: "expect_outbound", + label: "turn/interrupt/child", + frame: { + id: 4, + method: "turn/interrupt", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turnId: INTERRUPT_CHILD_NATIVE_TURN, + }, + }, + }, + { type: "emit_inbound", label: "turn/interrupt/child", frame: { id: 4, result: {} } }, + { + type: "emit_inbound", + label: "turn/completed/child", + frame: { + method: "turn/completed", + params: { + threadId: INTERRUPT_CHILD_NATIVE_THREAD, + turn: makeCodexReplayTurn({ + id: INTERRUPT_CHILD_NATIVE_TURN, + status: "interrupted", + }), + }, + }, + }, + ], + }); + const harness = yield* makeCodexReplayHarness(transcript); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("attempt-codex-rate-limit-subagent"), + text: INTERRUPT_PROMPT, + }), + ); + // The subagent update is the last event of the child's terminalization. + yield* harness.until(() => harness.subagentUpdates().at(-1)?.subagent.status === "failed"); + + const childRetryItems = harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.type === "error" && + event.turnItem.retry !== undefined && + event.turnItem.threadId !== harness.threadId + ? [event.turnItem] + : [], + ); + assert.deepEqual( + childRetryItems.map((item) => item.status), + ["running", "failed"], + ); + const [notice, exhausted] = childRetryItems; + assert.isDefined(notice); + assert.isDefined(exhausted); + assert.equal(exhausted.failure.code, "rateLimitRetryExhausted"); + assert.equal(exhausted.failure.class, "usage_limit"); + assert.equal(exhausted.failure.retryable, true); + assert.equal( + exhausted.failure.resetAt, + DateTime.formatIso(DateTime.add(notice.updatedAt, { milliseconds: 384_882_000 })), + ); + assert.deepEqual(exhausted.retry, { attempt: 1, maxAttempts: 5, retryDelayMs: null }); + assert.lengthOf(harness.terminalEvents(), 0, "the root turn keeps running"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ), + ); + it.effect.each([ { name: "usage", ``` ## apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts Status: modified; +826; −293 ```diff @@ -16,6 +16,14 @@ import type { CodexEffectiveRuntime } from "../../provider/CodexManagedRuntime.ts"; import { classifyCodexManagedError } from "../../provider/CodexManagedErrors.ts"; import { + CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS, + codexErrorHttpStatusCode, + codexErrorTextIndicatesHttp429, + decideCodexRateLimitRetry, + parseCodexRetryAfterMs, + type CodexRateLimitRetryDecision, +} from "../../provider/codexRateLimitRetry.ts"; +import { buildCodexInitializeParams, codexAppServerEnvironment, codexDefaultReasoningEffortOverride, @@ -211,6 +219,11 @@ // Each plan flush stores the whole plan three times (node, plan, turn item), so // flush at the cadence clients already see for assistant text. const CODEX_PLAN_DELTA_FLUSH_INTERVAL_MS = 400; +/** + * Bounds how long a turn the 429 deadline stops waits for Codex to confirm + * the stop, from sending turn/interrupt to its turn/completed. + */ +const CODEX_RATE_LIMIT_INTERRUPT_TIMEOUT_MS = 10_000; const CodexBackgroundTerminalTerminateResponse = Schema.Struct({ terminated: Schema.Boolean, }); @@ -1168,6 +1181,31 @@ readonly nativeCode: string | null; readonly failure: OrchestrationV2ProviderFailure; }; + /** + * Set once a stalled HTTP 429 retry is being stopped. The stopping fiber + * then owns the turn's terminal, so the interrupt's own turn/completed + * cannot replace the rate-limit failure. + */ + rateLimitRetryExhausted?: OrchestrationV2ProviderFailure; + /** + * Whether Codex confirmed the stalled 429 turn stopped. "ended" records a + * turn/completed that arrived while the interrupt was in flight or ended + * an unconfirmed turn; "unconfirmed" keeps the turn active until Codex + * reports it ended, for at most CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS; + * "abandoned" means that bound passed and the adapter failed the turn + * itself, or that Codex never confirmed with turn/completed that a + * descendant the deadline stopped with its root ended, so its native thread + * is treated as broken. + */ + rateLimitContainment?: "ended" | "unconfirmed" | "abandoned"; + /** + * Completed once Codex's turn/completed confirms a turn the 429 deadline + * interrupted ended. A turn/interrupt reply only accepts the request; the + * completion is the authoritative stop. + */ + rateLimitStopConfirmed?: Deferred.Deferred; + /** Fails an "unconfirmed" turn once its bound passes. */ + rateLimitContainmentTimer?: Fiber.Fiber; readonly nativeStartReady?: Deferred.Deferred; readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; readonly projectionAppThread: OrchestrationV2AppThread; @@ -1265,6 +1303,20 @@ return { owner, subagent }; }; +/** + * A 429 turn the adapter ended without Codex confirming it stopped may still + * run natively, so its thread must be resumed rather than reused. That holds + * whether the bound abandoned it or Stop ended it first, including while the + * deadline is still stopping it. A turn/interrupt reply only accepts the + * request; only turn/completed moves the turn to "ended". + */ +const isUncontainedRateLimitedTurn = (context: ActiveCodexTurnContext): boolean => + context.rateLimitContainment === "abandoned" || + context.rateLimitContainment === "unconfirmed" || + (context.rateLimitRetryExhausted !== undefined && + context.rateLimitContainment !== "ended" && + context.providerThread.nativeThreadRef?.nativeId != null); + const isDescendantCodexTurn = ( candidate: ActiveCodexTurnContext, ancestor: ActiveCodexTurnContext, @@ -2001,6 +2053,9 @@ context: ActiveCodexTurnContext, updatedAt: DateTime.Utc, ) { + // Once the deadline stopped the turn, the retry failed rather than + // recovered, and the turn's failure keeps its retry metadata. + if (context.rateLimitRetryExhausted !== undefined) return; const providerRetry = yield* Ref.modify(providerRetries, (current) => { const retry = current.get(context.providerTurnId); if (retry === undefined) { @@ -4585,7 +4640,9 @@ yield* client.handleServerNotification("error", (payload) => Effect.gen(function* () { const context = yield* awaitActiveTurn(payload.turnId); + // Once a stalled 429 retry is being stopped, the stopping fiber + // owns the failure, so a late error cannot replace it. + if (context === undefined || context.rateLimitRetryExhausted !== undefined) { - if (context === undefined) { return; } const notificationCode = codexErrorInfoCode(payload.error.codexErrorInfo); @@ -4609,13 +4666,31 @@ const updatedAt = yield* DateTime.now; const previous = (yield* Ref.get(providerRetries)).get(context.providerTurnId); const progress = parseCodexRetryProgress(payload.error.message); + const attempt = progress?.attempt ?? (previous?.retry.attempt ?? 0) + 1; + const code = codexErrorInfoCode(payload.error.codexErrorInfo); + const additionalDetails = payload.error.additionalDetails?.trim(); + const noticedAtMs = DateTime.toEpochMillis(updatedAt); + const retryAfterMs = + parseCodexRetryAfterMs(additionalDetails, noticedAtMs) ?? + parseCodexRetryAfterMs(payload.error.message, noticedAtMs); + // Codex sleeps for any Retry-After it gets, uncapped and silently, + // so a rate-limited retry is bounded here instead. + // The structured status is optional, so a 429 only named in the + // error text still counts when Codex attached no status at all. + const httpStatusCode = codexErrorHttpStatusCode(payload.error.codexErrorInfo); + const rateLimitDecision = + codexUsageLimitError(code, payload.error.codexErrorInfo) || + httpStatusCode === 429 || + (httpStatusCode === null && + (codexErrorTextIndicatesHttp429(additionalDetails) || + codexErrorTextIndicatesHttp429(payload.error.message))) + ? decideCodexRateLimitRetry({ attempt, retryAfterMs }) + : null; const retry: OrchestrationV2ProviderRetry = { - attempt: progress?.attempt ?? (previous?.retry.attempt ?? 0) + 1, + attempt, maxAttempts: progress?.maxAttempts ?? previous?.retry.maxAttempts ?? null, + retryDelayMs: rateLimitDecision?.type === "wait" ? rateLimitDecision.delayMs : null, - retryDelayMs: null, }; - const code = codexErrorInfoCode(payload.error.codexErrorInfo); - const additionalDetails = payload.error.additionalDetails?.trim(); const failure = withCodexManagedFailure( makeProviderFailure({ message: @@ -4666,9 +4741,387 @@ updatedAt, }), }); + if (rateLimitDecision !== null) { + yield* endStalledRateLimitedTurn({ + context, + state, + decision: rateLimitDecision, + retryAfterMs, + noticedAt: updatedAt, + failureAtArm: context.latestProviderFailure, + }).pipe(Effect.forkIn(scope)); + } }).pipe(Effect.orDie), ); + /** + * Sends turn/interrupt for a turn stopped by the 429 retry deadline. + * True only when Codex acknowledged it within the bound. + */ + const interruptRateLimitedNativeTurn = Effect.fnUntraced(function* ( + target: ActiveCodexTurnContext, + ) { + const nativeThreadId = target.providerThread.nativeThreadRef?.nativeId; + if (nativeThreadId == null) return true; + const acknowledged = yield* client + .request("turn/interrupt", { + threadId: nativeThreadId, + turnId: target.nativeTurnId, + }) + .pipe( + Effect.as(true), + Effect.timeoutOption(`${CODEX_RATE_LIMIT_INTERRUPT_TIMEOUT_MS} millis`), + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.codex-rate-limit-interrupt-failed", { + nativeTurnId: target.nativeTurnId, + cause, + }).pipe(Effect.as(Option.some(false))), + ), + ); + if (Option.isNone(acknowledged)) { + yield* Effect.logWarning("orchestration-v2.codex-rate-limit-interrupt-timeout", { + nativeTurnId: target.nativeTurnId, + }); + return false; + } + return acknowledged.value; + }); + + /** + * Interrupts a turn the 429 deadline stops and waits, within + * `timeoutMs`, for the turn/completed that confirms it ended. Codex + * replies to turn/interrupt before the turn ends, so the reply alone + * does not confirm the stop. True only when Codex confirmed it. + */ + const interruptRateLimitedTurnUntilEnded = Effect.fnUntraced(function* ( + target: ActiveCodexTurnContext, + timeoutMs: number, + ) { + const confirmed = yield* Deferred.make(); + target.rateLimitStopConfirmed = confirmed; + return yield* Effect.gen(function* () { + if (!(yield* interruptRateLimitedNativeTurn(target))) return false; + if ( + target.providerThread.nativeThreadRef?.nativeId == null || + target.rateLimitContainment === "ended" || + (yield* Ref.get(activeTurns)).get(target.nativeTurnId) !== target + ) { + return true; + } + yield* Deferred.await(confirmed); + return true; + }).pipe( + Effect.timeoutOption(`${timeoutMs} millis`), + Effect.map(Option.getOrElse(() => false)), + ); + }); + + /** + * Interrupts a descendant the 429 deadline stops with its root. A + * descendant Codex did not confirm ended may still run natively, so + * it is marked abandoned and its terminal publishes its thread as + * broken rather than idle. Once `timeoutMs` is spent, the interrupt + * is still sent but not waited on. + */ + const interruptRateLimitedDescendant = Effect.fnUntraced(function* ( + descendant: ActiveCodexTurnContext, + timeoutMs: number, + ) { + if (timeoutMs <= 0) { + yield* interruptRateLimitedNativeTurn(descendant).pipe(Effect.forkIn(scope)); + } else if (yield* interruptRateLimitedTurnUntilEnded(descendant, timeoutMs)) { + return; + } + if (descendant.rateLimitContainment !== "ended") { + descendant.rateLimitContainment = "abandoned"; + } + }); + + /** + * Interrupts the descendants found by one lineage scan at once. All + * scans of a lineage share one interrupt bound that ends at + * `deadlineAtMs`, so silent descendants, and nested turns found by a + * re-scan, cannot each add a full bound before the root settles. + */ + const interruptRateLimitedDescendants = Effect.fnUntraced(function* ( + descendants: ReadonlyArray, + deadlineAtMs: number, + ) { + const remainingMs = Math.max( + 0, + deadlineAtMs - DateTime.toEpochMillis(yield* DateTime.now), + ); + yield* Effect.forEach( + descendants, + (descendant) => interruptRateLimitedDescendant(descendant, remainingMs), + { concurrency: "unbounded", discard: true }, + ); + }); + + /** + * Ends a turn whose rate-limited retry made no progress within the + * retry budget. Progress or another retry notice replaces the tracked + * retry state, which turns this into a no-op. + */ + const endStalledRateLimitedTurn = Effect.fn("CodexAdapterV2.endStalledRateLimitedTurn")( + function* (input: { + readonly context: ActiveCodexTurnContext; + readonly state: ActiveCodexProviderRetry; + readonly decision: CodexRateLimitRetryDecision; + readonly retryAfterMs: number | null; + readonly noticedAt: DateTime.Utc; + readonly failureAtArm: ActiveCodexTurnContext["latestProviderFailure"]; + }) { + const { context, state, decision } = input; + if (decision.type === "wait") { + yield* Effect.sleep(`${decision.deadlineMs} millis`); + } + if ( + context.rateLimitRetryExhausted !== undefined || + (yield* Ref.get(providerRetries)).get(context.providerTurnId) !== state || + (yield* Ref.get(activeTurns)).get(context.nativeTurnId) !== context || + context.latestProviderFailure !== input.failureAtArm + ) { + return; + } + const now = yield* DateTime.now; + // A Retry-After that outlasts this wait becomes the limit's reset + // time, so the existing Limited state and auto-resume can use it. + const resetAt = + input.retryAfterMs === null + ? null + : DateTime.add(input.noticedAt, { milliseconds: input.retryAfterMs }); + const attempts = + state.retry.maxAttempts === null + ? `${state.retry.attempt}` + : `${state.retry.attempt}/${state.retry.maxAttempts}`; + const message = + decision.type === "wait" + ? `Provider rate limit (HTTP 429): retry ${attempts} made no progress within ${Math.round(decision.deadlineMs / 1000)}s, so the turn was stopped.` + : decision.reason === "attempts_exhausted" + ? `Provider rate limit (HTTP 429): gave up after ${attempts} retry attempts.` + : `Provider rate limit (HTTP 429): the provider asked to wait ${Math.round((input.retryAfterMs ?? 0) / 1000)}s before retrying, longer than the allowed retry delay, so the turn was stopped.`; + const failure = makeProviderFailure({ + message, + code: "rateLimitRetryExhausted", + class: "usage_limit", + retryable: true, + ...(resetAt === null || !DateTime.isGreaterThan(resetAt, now) + ? {} + : { resetAt: DateTime.formatIso(resetAt) }), + }); + context.latestProviderFailure = { + nativeMessage: state.nativeMessage, + nativeCode: state.nativeCode, + failure, + }; + context.rateLimitRetryExhausted = failure; + // Stop Codex's own retry sleep before publishing the terminal: once + // the turn is terminal the thread is idle, and a queued follow-up + // must not start while Codex still runs the rate-limited turn. + // Codex replies to turn/interrupt before the turn ends; only its + // turn/completed confirms the stop, so the root waits for it + // within the interrupt bound once its lineage is stopped. + const rootInterruptSentAt = yield* DateTime.now; + const rootStopConfirmed = yield* Deferred.make(); + context.rateLimitStopConfirmed = rootStopConfirmed; + const rootAcknowledged = yield* interruptRateLimitedNativeTurn(context); + let descendantsDeadlineAtMs: number | undefined; + const rootHasNativeThread = context.providerThread.nativeThreadRef?.nativeId != null; + let rootStopAwaited = false; + // Native subagents run on their own threads and outlive a failed + // root, so the deadline stops the whole active lineage, like Stop. + // A subagent can start a nested turn while an interrupt is in + // flight, so the lineage is re-scanned until no new turn appears. + // Descendants settle first, so the root terminal neither waits for + // them nor reports them as still running. + // The last, empty scan and the root's terminal share one + // terminalization permit with turn/started: a nested turn starts + // either before the scan, which then stops it, or after the root + // is terminal, where the late-subagent path interrupts it. + const stoppedDescendants = new Set(); + while (true) { + const step = yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + const descendants = Array.from((yield* Ref.get(activeTurns)).values()).filter( + (candidate) => + !stoppedDescendants.has(candidate) && + isDescendantCodexTurn(candidate, context), + ); + if (descendants.length > 0) { + return { type: "descendants", descendants } as const; + } + // An interrupt Codex did not confirm with turn/completed + // may not have stopped Codex's retry, so the turn stays + // active until its turn/completed confirms it ended. + if (rootHasNativeThread && context.rateLimitContainment !== "ended") { + if (rootAcknowledged && !rootStopAwaited) { + return { type: "awaitRootStop" } as const; + } + context.rateLimitContainment = "unconfirmed"; + return { type: "unconfirmed" } as const; + } + yield* finalizeCodexTurnHoldingPermit({ + context, + nativeTurnId: context.nativeTurnId, + status: "failed", + completedAt: now, + }); + return { type: "finalized" } as const; + }), + ); + if (step.type === "finalized") return; + if (step.type === "unconfirmed") break; + if (step.type === "awaitRootStop") { + // The wait is outside the permit, so turn/completed can end + // the root; the lineage is then scanned again. + rootStopAwaited = true; + const elapsedMs = + DateTime.toEpochMillis(yield* DateTime.now) - + DateTime.toEpochMillis(rootInterruptSentAt); + yield* Deferred.await(rootStopConfirmed).pipe( + Effect.timeoutOption( + `${Math.max(0, CODEX_RATE_LIMIT_INTERRUPT_TIMEOUT_MS - elapsedMs)} millis`, + ), + ); + continue; + } + for (const descendant of step.descendants) stoppedDescendants.add(descendant); + descendantsDeadlineAtMs ??= + DateTime.toEpochMillis(yield* DateTime.now) + CODEX_RATE_LIMIT_INTERRUPT_TIMEOUT_MS; + yield* interruptRateLimitedDescendants(step.descendants, descendantsDeadlineAtMs); + // A descendant can start after the deadline stopped the root, so + // it settles at the time its interrupt finished, not the + // deadline's, which would end it before it started. + const completedAt = yield* DateTime.now; + for (const descendant of step.descendants) { + yield* finalizeCodexTurn({ + context: descendant, + nativeTurnId: descendant.nativeTurnId, + status: "interrupted", + completedAt, + }); + } + } + yield* Effect.logWarning("orchestration-v2.codex-rate-limit-turn-uncontained", { + nativeTurnId: context.nativeTurnId, + }); + // Waiting for turn/completed alone could hang the turn again, so + // the wait is bounded. + context.rateLimitContainmentTimer = yield* abandonUncontainedRateLimitedTurn( + context, + ).pipe(Effect.forkIn(scope)); + }, + ); + + /** + * Stops the active lineage of a root the 429 deadline stopped, then + * settles the root. The lineage is re-scanned until no new nested + * turn appears; the last, empty scan and settle share one + * terminalization permit with turn/started, so a nested turn either is + * stopped here or starts after the root is terminal, where the + * late-subagent path interrupts it. Descendants settle first, so the + * root terminal neither waits for them nor reports them as running. + */ + const stopRateLimitedLineageThenSettle = Effect.fn( + "CodexAdapterV2.stopRateLimitedLineageThenSettle", + )(function* (input: { + readonly context: ActiveCodexTurnContext; + /** Checked holding the permit; false stops without settling. */ + readonly isPending: Effect.Effect; + /** Runs holding the permit once no unstopped descendant is active. */ + readonly settle: Effect.Effect; + }) { + const stoppedDescendants = new Set(); + let descendantsDeadlineAtMs: number | undefined; + while (true) { + const descendants = yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if (!(yield* input.isPending)) return []; + const descendants = Array.from((yield* Ref.get(activeTurns)).values()).filter( + (candidate) => + !stoppedDescendants.has(candidate) && + isDescendantCodexTurn(candidate, input.context), + ); + if (descendants.length === 0) yield* input.settle; + return descendants; + }), + ); + if (descendants.length === 0) return; + for (const descendant of descendants) stoppedDescendants.add(descendant); + descendantsDeadlineAtMs ??= + DateTime.toEpochMillis(yield* DateTime.now) + CODEX_RATE_LIMIT_INTERRUPT_TIMEOUT_MS; + yield* interruptRateLimitedDescendants(descendants, descendantsDeadlineAtMs); + const completedAt = yield* DateTime.now; + for (const descendant of descendants) { + yield* finalizeCodexTurn({ + context: descendant, + nativeTurnId: descendant.nativeTurnId, + status: "interrupted", + completedAt, + }); + } + } + }); + + /** + * Fails a stopped 429 turn whose stop Codex never confirmed. The turn + * may still be running natively, so the terminal marks its thread + * broken and the next turn resumes it rather than reusing it. + */ + const abandonUncontainedRateLimitedTurn = Effect.fn( + "CodexAdapterV2.abandonUncontainedRateLimitedTurn", + )(function* (context: ActiveCodexTurnContext) { + yield* Effect.sleep(`${CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS} millis`); + // A subagent can start a nested turn while the root waits, so the + // lineage is stopped again before the root fails, as the deadline + // did. A turn/completed either ends the turn first or is ignored. + // The bound caps the whole lineage: waiting on each nested turn's + // interrupt here would let one silent subagent stretch it, so the + // interrupts are sent without waiting, the nested turns and the root + // settle at the bound under one terminalization permit, and the + // nested turns' threads publish as broken because no stop was + // confirmed. Holding the permit keeps a nested turn from starting + // in between; one that starts later takes the late-subagent path. + yield* turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if ( + context.rateLimitContainment !== "unconfirmed" || + (yield* Ref.get(activeTurns)).get(context.nativeTurnId) !== context + ) { + return; + } + context.rateLimitContainment = "abandoned"; + yield* Effect.logWarning("orchestration-v2.codex-rate-limit-turn-abandoned", { + nativeTurnId: context.nativeTurnId, + timeoutMs: CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS, + }); + const completedAt = yield* DateTime.now; + const descendants = Array.from((yield* Ref.get(activeTurns)).values()).filter( + (candidate) => isDescendantCodexTurn(candidate, context), + ); + for (const descendant of descendants) { + if (descendant.rateLimitContainment !== "ended") { + descendant.rateLimitContainment = "abandoned"; + } + yield* interruptRateLimitedNativeTurn(descendant).pipe(Effect.forkIn(scope)); + yield* finalizeCodexTurnHoldingPermit({ + context: descendant, + nativeTurnId: descendant.nativeTurnId, + status: "interrupted", + completedAt, + }); + } + yield* finalizeCodexTurnHoldingPermit({ + context, + nativeTurnId: context.nativeTurnId, + status: "failed", + completedAt, + }); + }), + ); + }); + const emitCompactionItem = Effect.fn("CodexAdapterV2.emitCompactionItem")(function* ( context: ActiveCodexTurnContext, nativeItemId: string, @@ -4753,6 +5206,9 @@ } if (payload.item.type === "contextCompaction") { + // Compaction is a model request too, so it proves a retried + // request went through and must clear the retry deadline. + yield* completeProviderRetry(context, yield* DateTime.now); yield* emitCompactionItem( context, payload.item.id, @@ -4913,6 +5369,7 @@ } if (payload.item.type === "contextCompaction") { + yield* completeProviderRetry(context, yield* DateTime.now); yield* emitCompactionItem( context, payload.item.id, @@ -5719,7 +6176,9 @@ failure.class === "usage_limit" ? { ...failure, + resetAt: + failure.resetAt ?? - resetAt: codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)), + codexUsageLimitResetAt(yield* Ref.get(rateLimitSnapshot)), } : failure, ...(input.providerRetry === undefined @@ -5728,7 +6187,11 @@ retry: input.providerRetry.retry, retryStartedAt: input.providerRetry.startedAt, }), + // A 429 turn Codex never confirmed stopped must not reuse the + // loaded thread: "broken" makes the next turn resume it. + threadDisposition: isUncontainedRateLimitedTurn(input.context) + ? "broken" + : "reusable", - threadDisposition: "reusable", }; } return { @@ -5739,7 +6202,10 @@ runOrdinal: input.context.input.runOrdinal, status: terminalStatus, failure: null, + // Stop can end a 429 turn Codex never confirmed stopped. - threadDisposition: "reusable", + threadDisposition: isUncontainedRateLimitedTurn(input.context) + ? "broken" + : "reusable", }; }, ); @@ -5898,310 +6364,377 @@ }, ); - const finalizeCodexTurn = (input: { + interface FinalizeCodexTurnInput { readonly context: ActiveCodexTurnContext; readonly nativeTurnId: string; readonly status: OrchestrationV2ProviderTurn["status"]; readonly completedAt: DateTime.Utc; readonly failureMessage?: string; readonly failureErrorInfo?: CodexErrorInfo; + } + const finalizeCodexTurn = (input: FinalizeCodexTurnInput) => + turnTerminalizationPermit.withPermits(1)(finalizeCodexTurnHoldingPermit(input)); + /** The body of finalizeCodexTurn, for callers that hold the permit. */ + const finalizeCodexTurnHoldingPermit = Effect.fn( + "CodexAdapterV2.finalizeCodexTurnHoldingPermit", + )(function* (input: FinalizeCodexTurnInput) { + const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId); + if (current !== input.context) { + return false; + } + input.context.fileChangeKindsByItem.clear(); + const providerRetry = yield* Ref.modify(providerRetries, (current) => { + const retry = current.get(input.context.providerTurnId); + if (retry === undefined) { + return [undefined, current] as const; + } + const updated = new Map(current); + updated.delete(input.context.providerTurnId); + return [retry, updated] as const; + }); + if ( + providerRetry !== undefined && + (input.status !== "failed" || input.context.subagent !== null) + ) { + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CODEX_PROVIDER, + turnItem: makeProviderRetryTurnItem({ + driver: CODEX_PROVIDER, + threadId: input.context.projectionThreadId, + runId: input.context.projectionRunId, + nodeId: input.context.providerNodeId, + providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.providerTurnId, + itemOrdinal: providerRetry.itemOrdinal, + // A subagent has no root terminal, so its retry item is + // where a stalled 429's usage-limit failure shows. + failure: + input.status === "failed" && input.context.rateLimitRetryExhausted !== undefined + ? input.context.rateLimitRetryExhausted + : providerRetry.failure, + retry: providerRetry.retry, + status: providerTurnStatusToTerminal(input.status), + startedAt: providerRetry.startedAt, + updatedAt: input.completedAt, + }), + }); + } + if (input.status !== "completed") { + yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => { + const updated = new Set(current); + updated.add(input.nativeTurnId); + return updated; + }); + } + yield* agentMessageDeltas.flushTurn(input.nativeTurnId); + yield* reasoningDeltas.flushTurn(input.nativeTurnId); + yield* planDeltas.flushTurn(input.nativeTurnId); + for (const [key, part] of reasoningParts) { + if (part.turnId === input.nativeTurnId) reasoningParts.delete(key); + } + // Failed compaction can end the turn without item/completed. + for (const nativeItemId of input.context.runningCompactionItemIds) { + yield* emitCompactionItem( + input.context, + nativeItemId, + providerTurnStatusToTerminal(input.status), + input.completedAt, + ); + } + const nativeThreadId = input.context.providerThread.nativeThreadRef?.nativeId; + const activation = + nativeThreadId == null ? undefined : goalActivations.get(nativeThreadId); + const holdsForGoal = + input.context.subagent === null && - }) => - turnTerminalizationPermit.withPermits(1)( - Effect.gen(function* () { - const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId); - if (current !== input.context) { - return false; - } - input.context.fileChangeKindsByItem.clear(); - const providerRetry = yield* Ref.modify(providerRetries, (current) => { - const retry = current.get(input.context.providerTurnId); - if (retry === undefined) { - return [undefined, current] as const; - } - const updated = new Map(current); - updated.delete(input.context.providerTurnId); - return [retry, updated] as const; - }); - if ( - providerRetry !== undefined && - (input.status !== "failed" || input.context.subagent !== null) - ) { - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CODEX_PROVIDER, - turnItem: makeProviderRetryTurnItem({ - driver: CODEX_PROVIDER, - threadId: input.context.projectionThreadId, - runId: input.context.projectionRunId, - nodeId: input.context.providerNodeId, - providerThreadId: input.context.providerThread.id, - providerTurnId: input.context.providerTurnId, - itemOrdinal: providerRetry.itemOrdinal, - failure: providerRetry.failure, - retry: providerRetry.retry, - status: providerTurnStatusToTerminal(input.status), - startedAt: providerRetry.startedAt, - updatedAt: input.completedAt, - }), - }); - } - if (input.status !== "completed") { - yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => { - const updated = new Set(current); - updated.add(input.nativeTurnId); - return updated; - }); - } - yield* agentMessageDeltas.flushTurn(input.nativeTurnId); - yield* reasoningDeltas.flushTurn(input.nativeTurnId); - yield* planDeltas.flushTurn(input.nativeTurnId); - for (const [key, part] of reasoningParts) { - if (part.turnId === input.nativeTurnId) reasoningParts.delete(key); - } - // Failed compaction can end the turn without item/completed. - for (const nativeItemId of input.context.runningCompactionItemIds) { - yield* emitCompactionItem( - input.context, - nativeItemId, - providerTurnStatusToTerminal(input.status), - input.completedAt, - ); - } - const nativeThreadId = input.context.providerThread.nativeThreadRef?.nativeId; - const activation = - nativeThreadId == null ? undefined : goalActivations.get(nativeThreadId); - const holdsForGoal = - input.context.subagent === null && - input.status === "completed" && - nativeThreadId != null && - (goalsByNativeThread.get(nativeThreadId)?.status === "active" || + input.status === "completed" && + nativeThreadId != null && + (goalsByNativeThread.get(nativeThreadId)?.status === "active" || + (activation !== undefined && !activation.stopped)); + const completedTurn: OrchestrationV2ProviderTurn = { + id: input.context.providerTurnId, + providerThreadId: input.context.providerThread.id, + nodeId: input.context.providerNodeId, + runAttemptId: input.context.subagent === null ? input.context.input.attemptId : null, + nativeTurnRef: { + driver: CODEX_PROVIDER, + nativeId: input.nativeTurnId, + strength: "strong", + }, + ordinal: input.context.providerTurnOrdinal, + status: input.status, + startedAt: input.context.startedAt, + completedAt: input.completedAt, + turnTokenUsage: completeCodexTurnTokenUsage( + usageStateForThread( + input.context.providerThread.nativeThreadRef?.nativeId ?? + String(input.context.providerThread.id), + ), + input.nativeTurnId, + input.status === "completed", + ), + }; - (activation !== undefined && !activation.stopped)); - const completedTurn: OrchestrationV2ProviderTurn = { - id: input.context.providerTurnId, + if (!holdsForGoal) { + yield* emitProviderEvent({ + type: "provider_turn.updated", + driver: CODEX_PROVIDER, + threadId: input.context.projectionThreadId, + providerTurn: completedTurn, + }); + } + if (input.context.subagent !== null) { + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node: { + id: input.context.providerNodeId, + threadId: input.context.projectionThreadId, + runId: null, + parentNodeId: null, + rootNodeId: input.context.rootNodeId, + kind: "root_turn", + status: input.status, + countsForRun: false, providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.providerTurnId, + nativeItemRef: input.context.subagent.task.nativeTaskRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: input.context.providerNodeStartedAt, + completedAt: input.completedAt, + }, - nodeId: input.context.providerNodeId, + }); + // A subagent has no root terminal to carry a broken disposition, + // so a 429 turn Codex never confirmed stopped marks its thread - runAttemptId: - input.context.subagent === null ? input.context.input.attemptId : null, - nativeTurnRef: { - driver: CODEX_PROVIDER, - nativeId: input.nativeTurnId, - strength: "strong", - }, - ordinal: input.context.providerTurnOrdinal, + // errored: the child identity then resumes it instead of reusing + // a thread that may still be running. + yield* emitProviderEvent({ + type: "provider_thread.updated", + driver: CODEX_PROVIDER, + providerThread: { + ...input.context.providerThread, + status: isUncontainedRateLimitedTurn(input.context) ? "error" : "idle", + updatedAt: input.completedAt, + }, + }); + yield* emitProviderEvent({ + type: "node.updated", + driver: CODEX_PROVIDER, + node: { + id: input.context.subagent.subagentNodeId, + threadId: input.context.subagent.parentContext.projectionThreadId, + runId: input.context.subagent.parentContext.projectionRunId, + parentNodeId: input.context.subagent.parentContext.itemParentNodeId, + rootNodeId: input.context.subagent.parentContext.rootNodeId, + kind: "subagent", status: input.status, + countsForRun: false, + providerThreadId: input.context.providerThread.id, + providerTurnId: input.context.subagent.parentContext.providerTurnId, + nativeItemRef: input.context.subagent.task.nativeTaskRef, + runtimeRequestId: null, + checkpointScopeId: null, - startedAt: input.context.startedAt, + startedAt: input.context.subagent.startedAt, completedAt: input.completedAt, - turnTokenUsage: completeCodexTurnTokenUsage( - usageStateForThread( - input.context.providerThread.nativeThreadRef?.nativeId ?? - String(input.context.providerThread.id), - ), - input.nativeTurnId, - input.status === "completed", - ), - }; - if (!holdsForGoal) { - yield* emitProviderEvent({ - type: "provider_turn.updated", - driver: CODEX_PROVIDER, - threadId: input.context.projectionThreadId, - providerTurn: completedTurn, - }); - } - if (input.context.subagent !== null) { - yield* emitProviderEvent({ - type: "node.updated", - driver: CODEX_PROVIDER, - node: { - id: input.context.providerNodeId, - threadId: input.context.projectionThreadId, - runId: null, - parentNodeId: null, - rootNodeId: input.context.rootNodeId, - kind: "root_turn", - status: input.status, - countsForRun: false, - providerThreadId: input.context.providerThread.id, - providerTurnId: input.context.providerTurnId, - nativeItemRef: input.context.subagent.task.nativeTaskRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: input.context.providerNodeStartedAt, - completedAt: input.completedAt, - }, - }); - yield* emitProviderEvent({ - type: "provider_thread.updated", + }, + }); + yield* emitSubagentTaskUpdate({ + subagent: input.context.subagent, + status: input.status, + completedAt: input.completedAt, + }); + // Codex does not wake a parent whose turn already completed, so + // the child's result would sit unread until the next message. + // Only direct children of a root turn wake it: a grandchild + // reports to its own parent, and a root turn already running on + // the thread receives the result natively. + const parent = input.context.subagent.parentContext; + if ( + continuationRequests !== undefined && + parent.subagent === null && + (input.status === "completed" || input.status === "failed") && + !(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(parent.nativeTurnId) && - driver: CODEX_PROVIDER, + ![...(yield* Ref.get(activeTurns)).values()].some( - providerThread: { - ...input.context.providerThread, - status: "idle", - updatedAt: input.completedAt, - }, - }); - yield* emitProviderEvent({ - type: "node.updated", - driver: CODEX_PROVIDER, - node: { - id: input.context.subagent.subagentNodeId, - threadId: input.context.subagent.parentContext.projectionThreadId, - runId: input.context.subagent.parentContext.projectionRunId, - parentNodeId: input.context.subagent.parentContext.itemParentNodeId, - rootNodeId: input.context.subagent.parentContext.rootNodeId, + (active) => + active.subagent === null && active.providerThread.id === parent.providerThread.id, + ) + ) { + const task = input.context.subagent.task; + yield* continuationRequests.offer({ + threadId: parent.projectionThreadId, + providerThreadId: parent.providerThread.id, + driver: CODEX_PROVIDER, + detail: codexNativeSubagentDetail(task), + // The detail is the whole wake; Codex buffers no output for it. + delivery: "message_text", + notification: backgroundWorkNotification([ + { kind: "subagent", - status: input.status, + label: codexNativeSubagentLabel(task), + outcome: input.status, + childThreadId: input.context.subagent.childThreadId, - countsForRun: false, - providerThreadId: input.context.providerThread.id, - providerTurnId: input.context.subagent.parentContext.providerTurnId, - nativeItemRef: input.context.subagent.task.nativeTaskRef, - runtimeRequestId: null, - checkpointScopeId: null, - startedAt: input.context.subagent.startedAt, - completedAt: input.completedAt, }, + ]), + }); + } + } + if (input.status === "interrupted" || input.status === "failed") { + yield* terminalizeRunningCommandItems( + input.context, + input.nativeTurnId, + input.status, - }); - yield* emitSubagentTaskUpdate({ - subagent: input.context.subagent, - status: input.status, - completedAt: input.completedAt, - }); - // Codex does not wake a parent whose turn already completed, so - // the child's result would sit unread until the next message. - // Only direct children of a root turn wake it: a grandchild - // reports to its own parent, and a root turn already running on - // the thread receives the result natively. - const parent = input.context.subagent.parentContext; - if ( - continuationRequests !== undefined && - parent.subagent === null && - (input.status === "completed" || input.status === "failed") && - !(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(parent.nativeTurnId) && - ![...(yield* Ref.get(activeTurns)).values()].some( - (active) => - active.subagent === null && - active.providerThread.id === parent.providerThread.id, - ) - ) { - const task = input.context.subagent.task; - yield* continuationRequests.offer({ - threadId: parent.projectionThreadId, - providerThreadId: parent.providerThread.id, - driver: CODEX_PROVIDER, - detail: codexNativeSubagentDetail(task), - // The detail is the whole wake; Codex buffers no output for it. - delivery: "message_text", - notification: backgroundWorkNotification([ - { - kind: "subagent", - label: codexNativeSubagentLabel(task), + input.completedAt, + ); + yield* terminalizeMcpAppCaptures(input.context, input.nativeTurnId); + } + const dynamicToolStatus: "cancelled" | "interrupted" | "failed" = + input.status === "interrupted" || input.status === "failed" + ? input.status + : "cancelled"; + yield* terminalizeRunningDynamicTools( + input.context, + input.nativeTurnId, + dynamicToolStatus, + input.completedAt, + input.status === "interrupted" || input.status === "failed", + ); + if (input.context.subagent === null && isUncontainedRateLimitedTurn(input.context)) { + // The broken disposition only reaches the projection. A live + // session keeps the native thread loaded until it sees the thread + // errored, and would otherwise start the next turn on a thread + // that may still be retrying. The update replaces the whole + // thread, so it starts from the live thread, whose goal may have + // changed since the turn started. + const nativeThreadId = input.context.providerThread.nativeThreadRef?.nativeId; + const liveThread = + (nativeThreadId == null ? undefined : rootProviderThreads.get(nativeThreadId)) ?? + input.context.providerThread; + yield* emitProviderEvent({ - outcome: input.status, - childThreadId: input.context.subagent.childThreadId, - }, - ]), - }); - } + type: "provider_thread.updated", + driver: CODEX_PROVIDER, + providerThread: { ...liveThread, status: "error", updatedAt: input.completedAt }, + }); + } + if (input.context.subagent === null) { + yield* emitOrDeferRootTerminal({ + ...input, + ...(providerRetry === undefined ? {} : { providerRetry }), + ...(holdsForGoal ? { goalHoldTurn: completedTurn } : {}), + }); + } + const waiter = (yield* Ref.get(turnWaiters)).get(input.nativeTurnId); + if (waiter !== undefined) { + yield* Deferred.succeed(waiter, undefined); + } + if (input.context.rateLimitStopConfirmed !== undefined) { + yield* Deferred.succeed(input.context.rateLimitStopConfirmed, undefined); + } + const interruptInProgress = (yield* Ref.get(interruptingNativeTurns)).has( + input.nativeTurnId, + ); + const retainSettledContext = + input.status === "completed" && + (yield* turnHasRetainedBackgroundWork(input.nativeTurnId)); + if (retainSettledContext) { + yield* Ref.update(settledTurns, (current) => { + const updated = new Map(current); + updated.set(input.nativeTurnId, input.context); + return updated; + }); + // Background work that finished between the check above and + // the turn being recorded as settled saw no settled turn to + // release, so it is re-checked once recorded. + yield* releaseSettledTurnIfIdle(input.nativeTurnId); + } + yield* Ref.update(activeTurns, (current) => { + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + if (input.context.nativeStartReady !== undefined) { + yield* Deferred.succeed(input.context.nativeStartReady, undefined); + } + yield* flushReadyRootTerminals(); + if (!retainSettledContext && !interruptInProgress) { + yield* Ref.update(runningCommandItemsByTurn, (current) => { + if (!current.has(input.nativeTurnId)) { + return current; } + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + yield* Ref.update(runningDynamicToolsByTurn, (current) => { + if (!current.has(input.nativeTurnId)) { - if (input.status === "interrupted" || input.status === "failed") { + return current; - yield* terminalizeRunningCommandItems( - input.context, - input.nativeTurnId, - input.status, - input.completedAt, - ); - yield* terminalizeMcpAppCaptures(input.context, input.nativeTurnId); } + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + } + if (!retainSettledContext) { + yield* Ref.update(completedFinalAnswerTextsByTurn, (current) => { + if (!current.has(input.nativeTurnId)) { - const dynamicToolStatus: "cancelled" | "interrupted" | "failed" = - input.status === "interrupted" || input.status === "failed" - ? input.status - : "cancelled"; - yield* terminalizeRunningDynamicTools( - input.context, - input.nativeTurnId, - dynamicToolStatus, - input.completedAt, - input.status === "interrupted" || input.status === "failed", - ); - if (input.context.subagent === null) { - yield* emitOrDeferRootTerminal({ - ...input, - ...(providerRetry === undefined ? {} : { providerRetry }), - ...(holdsForGoal ? { goalHoldTurn: completedTurn } : {}), - }); - } - const waiter = (yield* Ref.get(turnWaiters)).get(input.nativeTurnId); - if (waiter !== undefined) { - yield* Deferred.succeed(waiter, undefined); - } - const interruptInProgress = (yield* Ref.get(interruptingNativeTurns)).has( - input.nativeTurnId, - ); - const retainSettledContext = - input.status === "completed" && - (yield* turnHasRetainedBackgroundWork(input.nativeTurnId)); - if (retainSettledContext) { - yield* Ref.update(settledTurns, (current) => { - const updated = new Map(current); - updated.set(input.nativeTurnId, input.context); - return updated; - }); - // Background work that finished between the check above and - // the turn being recorded as settled saw no settled turn to + return current; - // release, so it is re-checked once recorded. - yield* releaseSettledTurnIfIdle(input.nativeTurnId); } - yield* Ref.update(activeTurns, (current) => { - const updated = new Map(current); - updated.delete(input.nativeTurnId); - return updated; - }); + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { - if (input.context.nativeStartReady !== undefined) { - yield* Deferred.succeed(input.context.nativeStartReady, undefined); + if (!current.has(input.nativeTurnId)) { + return current; } + const updated = new Map(current); + updated.delete(input.nativeTurnId); + return updated; + }); + } - yield* flushReadyRootTerminals(); - if (!retainSettledContext && !interruptInProgress) { + return true; + }); - yield* Ref.update(runningCommandItemsByTurn, (current) => { - if (!current.has(input.nativeTurnId)) { - return current; - } - const updated = new Map(current); - updated.delete(input.nativeTurnId); - return updated; - }); - yield* Ref.update(runningDynamicToolsByTurn, (current) => { - if (!current.has(input.nativeTurnId)) { - return current; - } - const updated = new Map(current); - updated.delete(input.nativeTurnId); - return updated; - }); - } - if (!retainSettledContext) { - yield* Ref.update(completedFinalAnswerTextsByTurn, (current) => { - if (!current.has(input.nativeTurnId)) { - return current; - } - const updated = new Map(current); - updated.delete(input.nativeTurnId); - return updated; - }); - yield* Ref.update(finalAnswerItemIdsByTurn, (current) => { - if (!current.has(input.nativeTurnId)) { - return current; - } - const updated = new Map(current); - updated.delete(input.nativeTurnId); - return updated; - }); - } - return true; - }), - ); yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); + if (context === undefined) return; + // A turn stopped for a stalled 429 retry is finalized by the fiber + // that interrupted it, with the rate-limit failure, unless that + // interrupt went unacknowledged and only this event confirms it. + // Once the unconfirmed wait was abandoned, that fiber owns the + // terminal and this late event is ignored. + if (context.rateLimitRetryExhausted !== undefined) { + if (context.rateLimitContainment !== "unconfirmed") { + if (context.rateLimitContainment === undefined) { + context.rateLimitContainment = "ended"; + if (context.rateLimitStopConfirmed !== undefined) { + yield* Deferred.succeed(context.rateLimitStopConfirmed, undefined); + } + } - if (context === undefined) { + return; + } + context.rateLimitContainment = "ended"; + // A subagent can start a nested turn during the unconfirmed + // wait. It must not outlive the root, so the lineage is stopped + // before the root fails, as the deadline and the bound do. Its + // interrupts await Codex's replies, so it runs off this handler. + yield* Effect.gen(function* () { + yield* stopRateLimitedLineageThenSettle({ + context, + isPending: Ref.get(activeTurns).pipe( + Effect.map((turns) => turns.get(payload.turn.id) === context), + ), + settle: finalizeCodexTurnHoldingPermit({ + context, + nativeTurnId: payload.turn.id, + status: "failed", + completedAt: codexTimestamp(payload.turn.completedAt), + }).pipe(Effect.asVoid), + }); + if (context.rateLimitContainmentTimer !== undefined) { + yield* Fiber.interrupt(context.rateLimitContainmentTimer); + } + }).pipe(Effect.forkIn(scope)); return; } const nativeStatus = mapCodexTurnStatus(payload.turn.status); ``` ## apps/server/src/provider/codexRateLimitRetry.test.ts Status: added; +98; −0 ```diff @@ -1,0 +1,98 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + CODEX_RATE_LIMIT_RETRY_POLICY, + codexErrorHttpStatusCode, + codexErrorTextIndicatesHttp429, + codexRateLimitBackoffMs, + decideCodexRateLimitRetry, + parseCodexRetryAfterMs, +} from "./codexRateLimitRetry.ts"; + +describe("codexRateLimitRetry", () => { + it("backs off exponentially from the base delay up to the max delay", () => { + expect([1, 2, 3, 4, 5, 6, 7, 8].map((attempt) => codexRateLimitBackoffMs(attempt))).toEqual([ + 5_000, 10_000, 20_000, 40_000, 80_000, 160_000, 300_000, 300_000, + ]); + }); + + it("waits per Retry-After first, plus the progress grace", () => { + expect(decideCodexRateLimitRetry({ attempt: 1, retryAfterMs: 30_000 })).toEqual({ + type: "wait", + delayMs: 30_000, + deadlineMs: 30_000 + CODEX_RATE_LIMIT_RETRY_POLICY.progressGraceMs, + }); + expect(decideCodexRateLimitRetry({ attempt: 3, retryAfterMs: null })).toEqual({ + type: "wait", + delayMs: 20_000, + deadlineMs: 20_000 + CODEX_RATE_LIMIT_RETRY_POLICY.progressGraceMs, + }); + }); + + it("gives up past the attempt cap or a Retry-After longer than the max delay", () => { + expect(decideCodexRateLimitRetry({ attempt: 5, retryAfterMs: null }).type).toBe("wait"); + expect(decideCodexRateLimitRetry({ attempt: 6, retryAfterMs: null })).toEqual({ + type: "give_up", + reason: "attempts_exhausted", + }); + expect( + decideCodexRateLimitRetry({ + attempt: 1, + retryAfterMs: CODEX_RATE_LIMIT_RETRY_POLICY.maxDelayMs + 1, + }), + ).toEqual({ type: "give_up", reason: "retry_after_exceeds_max_delay" }); + }); + + it("reads retry delays from provider error text", () => { + expect(parseCodexRetryAfterMs("429 Too Many Requests; Retry-After: 384882")).toBe(384_882_000); + expect(parseCodexRetryAfterMs('{"retry_after": 1.5}')).toBe(1_500); + expect(parseCodexRetryAfterMs('{"error":{"reset_seconds":12}}')).toBe(12_000); + expect(parseCodexRetryAfterMs("Rate limited. Please try again in 2 minutes.")).toBe(120_000); + expect(parseCodexRetryAfterMs("Reconnecting... 1/5")).toBeNull(); + expect(parseCodexRetryAfterMs(null)).toBeNull(); + }); + + it("reads the unit a retry-after delay names", () => { + expect(parseCodexRetryAfterMs("Rate limited. Please retry after 2 minutes.")).toBe(120_000); + expect(parseCodexRetryAfterMs("Rate limited; retry after 500ms")).toBe(500); + expect(parseCodexRetryAfterMs("retry-after: 1 hour")).toBe(3_600_000); + expect(parseCodexRetryAfterMs("Retry-After: 30s")).toBe(30_000); + expect(parseCodexRetryAfterMs("retry-after-ms: 250")).toBe(250); + // A bare Retry-After value stays delay-seconds. + expect(parseCodexRetryAfterMs("Retry-After: 30; so retry later")).toBe(30_000); + }); + + it("reads an HTTP-date Retry-After as the delay until that date", () => { + const nowMs = Date.parse("2026-10-22T11:59:30Z"); + expect( + parseCodexRetryAfterMs( + "429 Too Many Requests; Retry-After: Thu, 22 Oct 2026 12:00:00 GMT", + nowMs, + ), + ).toBe(30_000); + expect(parseCodexRetryAfterMs('{"retry-after":"Thu, 22 Oct 2026 13:00:00 GMT"}', nowMs)).toBe( + 3_630_000, + ); + // A date already in the past allows an immediate retry. + expect(parseCodexRetryAfterMs("Retry-After: Thu, 22 Oct 2026 11:00:00 GMT", nowMs)).toBe(0); + }); + + it("recognizes HTTP 429 in provider error text", () => { + expect(codexErrorTextIndicatesHttp429("429 Too Many Requests; Retry-After: 30")).toBe(true); + expect(codexErrorTextIndicatesHttp429('{"status": 429, "error": "rate_limited"}')).toBe(true); + expect(codexErrorTextIndicatesHttp429("unexpected status code 429")).toBe(true); + expect(codexErrorTextIndicatesHttp429("HTTP 429")).toBe(true); + expect(codexErrorTextIndicatesHttp429("The response stream disconnected.")).toBe(false); + expect(codexErrorTextIndicatesHttp429("unexpected status 503; request id 4291")).toBe(false); + expect(codexErrorTextIndicatesHttp429(undefined)).toBe(false); + }); + + it("reads the HTTP status from any structured Codex error", () => { + expect(codexErrorHttpStatusCode({ responseStreamDisconnected: { httpStatusCode: 429 } })).toBe( + 429, + ); + expect(codexErrorHttpStatusCode({ httpConnectionFailed: { httpStatusCode: null } })).toBeNull(); + expect(codexErrorHttpStatusCode("usageLimitExceeded")).toBeNull(); + expect(codexErrorHttpStatusCode(null)).toBeNull(); + }); +}); ``` ## apps/server/src/provider/codexRateLimitRetry.ts Status: added; +172; −0 ```diff @@ -1,0 +1,172 @@ +/** + * Bounds how long a Codex turn may sit in a provider HTTP 429 retry. + * + * The Codex CLI retries rate-limited model requests itself and only tells the + * app server "Reconnecting... n/m" (`error` with `willRetry: true`). When the + * upstream sends a long `Retry-After`, Codex sleeps for the full value without + * any cap and emits nothing until it wakes, which leaves a turn (and any + * delegated task waiting on it) "running" for hours. The adapter cannot + * intercept the HTTP retry, so it applies this policy around it: every retry + * notice gets an exponential-backoff (or Retry-After) budget, and when that + * budget, the attempt cap, or the maximum delay is exceeded the turn ends as + * an explicit usage-limit failure instead of hanging. + */ +export interface CodexRateLimitRetryPolicy { + /** Highest retry attempt the adapter waits for. Later notices end the turn. */ + readonly maxAttempts: number; + /** Expected wait for attempt 1 when no Retry-After is known; doubles per attempt. */ + readonly baseDelayMs: number; + /** Longest single wait the adapter accepts, including a server Retry-After. */ + readonly maxDelayMs: number; + /** Extra time after the expected wait for the retried request to show progress. */ + readonly progressGraceMs: number; +} + +export const CODEX_RATE_LIMIT_RETRY_POLICY: CodexRateLimitRetryPolicy = { + maxAttempts: 5, + baseDelayMs: 5_000, + maxDelayMs: 5 * 60_000, + progressGraceMs: 60_000, +}; + +/** + * How long a stopped 429 turn may stay active after Codex failed to + * acknowledge its `turn/interrupt` (the request errored or timed out) and has + * not reported `turn/completed` either. Past this bound the adapter stops + * waiting: the turn fails with `rateLimitRetryExhausted` and the native thread + * is marked broken, so the next turn resumes it instead of reusing a loaded + * thread that may still be retrying. It matches `progressGraceMs`, the time + * the policy already allows Codex to show any sign of life after a retry. + */ +export const CODEX_RATE_LIMIT_UNCONFIRMED_STOP_TIMEOUT_MS = 60_000; + +export type CodexRateLimitRetryDecision = + | { + readonly type: "wait"; + /** How long the provider is expected to wait before retrying. */ + readonly delayMs: number; + /** How long the adapter waits for progress before ending the turn. */ + readonly deadlineMs: number; + } + | { + readonly type: "give_up"; + readonly reason: "attempts_exhausted" | "retry_after_exceeds_max_delay"; + }; + +export function codexRateLimitBackoffMs( + attempt: number, + policy: CodexRateLimitRetryPolicy = CODEX_RATE_LIMIT_RETRY_POLICY, +): number { + const exponent = Math.max(0, Math.floor(attempt) - 1); + return Math.min(policy.baseDelayMs * 2 ** exponent, policy.maxDelayMs); +} + +export function decideCodexRateLimitRetry( + input: { readonly attempt: number; readonly retryAfterMs: number | null }, + policy: CodexRateLimitRetryPolicy = CODEX_RATE_LIMIT_RETRY_POLICY, +): CodexRateLimitRetryDecision { + if (input.attempt > policy.maxAttempts) { + return { type: "give_up", reason: "attempts_exhausted" }; + } + if (input.retryAfterMs !== null && input.retryAfterMs > policy.maxDelayMs) { + return { type: "give_up", reason: "retry_after_exceeds_max_delay" }; + } + const delayMs = input.retryAfterMs ?? codexRateLimitBackoffMs(input.attempt, policy); + return { type: "wait", delayMs, deadlineMs: delayMs + policy.progressGraceMs }; +} + +const UNIT_MS: Record = { + ms: 1, + millisecond: 1, + milliseconds: 1, + s: 1_000, + sec: 1_000, + secs: 1_000, + second: 1_000, + seconds: 1_000, + m: 60_000, + min: 60_000, + mins: 60_000, + minute: 60_000, + minutes: 60_000, + h: 3_600_000, + hr: 3_600_000, + hrs: 3_600_000, + hour: 3_600_000, + hours: 3_600_000, +}; + +const DELAY_UNIT = String.raw`milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h`; + +const RETRY_AFTER_DELAY = new RegExp( + String.raw`retry[-_ ]?after([-_]ms)?["']?\s*[:=]?\s*["']?(\d+(?:\.\d+)?)(?:["']?[ \t]*(${DELAY_UNIT})\b)?`, + "iu", +); + +const RETRY_IN_PHRASE = new RegExp( + String.raw`(?:try again|retry) in\s+(\d+(?:\.\d+)?)\s*(${DELAY_UNIT})\b`, + "iu", +); + +const toDelayMs = (amount: string, unitMs: number | undefined): number | null => { + const value = Number.parseFloat(amount); + return unitMs === undefined || !Number.isFinite(value) ? null : Math.round(value * unitMs); +}; + +// RFC 9110 lets Retry-After carry an IMF-fixdate instead of delay-seconds. +const RETRY_AFTER_HTTP_DATE = + /retry[-_ ]?after["']?\s*[:=]?\s*["']?((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun),\s*\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{4}\s+\d{2}:\d{2}:\d{2}\s+GMT)/iu; + +/** + * Reads a retry delay from provider error text: a `Retry-After` header or + * field (seconds unless it names a unit, or an HTTP date measured from + * `nowMs` when the caller supplies it), a `retry-after-ms` header, a + * `reset_seconds` field, or "try again in N ". + */ +export function parseCodexRetryAfterMs( + text: string | null | undefined, + nowMs?: number, +): number | null { + if (text == null || text.length === 0) return null; + if (nowMs !== undefined) { + const httpDate = RETRY_AFTER_HTTP_DATE.exec(text)?.[1]; + const retryAt = httpDate === undefined ? Number.NaN : Date.parse(httpDate); + if (Number.isFinite(retryAt)) return Math.max(0, retryAt - nowMs); + } + const retryAfter = RETRY_AFTER_DELAY.exec(text); + if (retryAfter?.[2] !== undefined) { + // A bare value is delay-seconds; `retry-after-ms` and a named unit + // ("retry after 2 minutes") override it. + const unit = + retryAfter[1] !== undefined ? UNIT_MS.ms : UNIT_MS[retryAfter[3]?.toLowerCase() ?? "s"]; + return toDelayMs(retryAfter[2], unit); + } + const resetSeconds = /reset_seconds["']?\s*[:=]\s*["']?(\d+(?:\.\d+)?)/iu.exec(text)?.[1]; + if (resetSeconds !== undefined) return toDelayMs(resetSeconds, UNIT_MS.s); + const phrase = RETRY_IN_PHRASE.exec(text); + if (phrase?.[1] === undefined || phrase[2] === undefined) return null; + return toDelayMs(phrase[1], UNIT_MS[phrase[2].toLowerCase()]); +} + +/** The HTTP status Codex attached to a structured error, whatever its variant. */ +export function codexErrorHttpStatusCode(errorInfo: unknown): number | null { + if (typeof errorInfo !== "object" || errorInfo === null) return null; + for (const detail of Object.values(errorInfo)) { + if (typeof detail === "object" && detail !== null && "httpStatusCode" in detail) { + const status = (detail as { readonly httpStatusCode?: unknown }).httpStatusCode; + if (typeof status === "number") return status; + } + } + return null; +} + +const TEXTUAL_HTTP_429 = + /\btoo many requests\b|\b(?:http|status(?:[ _]?code)?)["']?\s*[:=]?\s*["']?429\b|^\s*429\b/imu; + +/** + * Whether provider error text reports HTTP 429. Codex's structured status is + * optional, so the text is the fallback evidence when it is missing. + */ +export function codexErrorTextIndicatesHttp429(text: string | null | undefined): boolean { + return text != null && TEXTUAL_HTTP_429.test(text); +} ```