apps/server/src/orchestration-v2/RestartContinuation.ts
import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
import {
CommandId,
latestProviderTurnForAttempt,
MessageId,
type OrchestrationV2Run,
type RunId,
type ThreadId,
} from "@t3tools/contracts";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import type { ProjectionRuntimeRecoveryState } from "./ProjectionStore.ts";
import * as ServerSettings from "../serverSettings.ts";
import { isNativeMaintenanceCommand } from "./Orchestrator.ts";
import * as ThreadManagementService from "./ThreadManagementService.ts";
import {
cancelledRosterTaskWork,
cancelledTurnItemWork,
isAwaitedWork,
isRestartContinuationSource,
isRestartNoteSource,
restartCancelledBackgroundWorkNote,
restartContinuationNote,
} from "./RestartBackgroundNote.ts";
const CONTINUE_PROMPT = "Continue where you left off.";
const MAX_CONTINUATION_CHAIN = 3;
function restartContinuationBlocked(
projection: Pick<
ProjectionRuntimeRecoveryState,
"thread" | "runtimeRequests" | "providerTurns" | "turnItems"
>,
source: OrchestrationV2Run,
now: DateTime.Utc,
settled: boolean,
): boolean {
const { thread } = projection;
if (
thread.archivedAt !== null ||
thread.deletedAt !== null ||
thread.settledOverride === "settled" ||
(thread.snoozedUntil != null && DateTime.isGreaterThan(thread.snoozedUntil, now))
)
return true;
const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId);
if (
!settled &&
turn !== undefined &&
(turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")
)
return true;
const waitingRequest = projection.runtimeRequests.some(
(request) =>
request.kind !== "dynamic_tool_call" &&
request.kind !== "auth_refresh" &&
request.responseCapability.type !== "message" &&
(request.status === "pending" ||
((request.status === "expired" || request.status === "cancelled") &&
request.responseCapability.type === "not_resumable" &&
turn !== undefined &&
turn.id === request.providerTurnId)),
);
if (waitingRequest) return true;
return projection.turnItems.some(
(item) =>
item.type === "secret_request" &&
item.runId === source.id &&
(item.secretStatus === "pending" ||
(item.secretStatus === "cancelled" &&
item.completedAt !== null &&
source.completedAt !== null &&
DateTime.Equivalence(item.completedAt, source.completedAt))),
);
}
/**
* App-owned delegations are not counted: their own threads continue and report
* back.
*/
function hasOpenAwaitedWork(
projection: Pick<ProjectionRuntimeRecoveryState, "runs" | "turnItems">,
providerThread: ProjectionRuntimeRecoveryState["providerThreads"][number],
): boolean {
const openItem = projection.turnItems.some((item) => {
if (item.status !== "pending" && item.status !== "running" && item.status !== "waiting")
return false;
if (item.type === "subagent" && item.origin === "app_owned" && item.childThreadId !== null)
return false;
const work = cancelledTurnItemWork(item);
const providerThreadId =
item.providerThreadId ??
projection.runs.find((run) => run.id === item.runId)?.providerThreadId;
return work !== undefined && isAwaitedWork(work) && providerThreadId === providerThread.id;
});
return (
openItem ||
(providerThread.ownerNodeId === null &&
(providerThread.pendingBackgroundTasks ?? []).some((task) =>
isAwaitedWork(cancelledRosterTaskWork(task)),
))
);
}
/**
* A settled run needs a live session, so stale records from an older crash
* never wake it.
*/
export function restartContinuationRun(
projection: Pick<
ProjectionRuntimeRecoveryState,
| "thread"
| "runs"
| "providerThreads"
| "providerSessions"
| "providerTurns"
| "runtimeRequests"
| "turnItems"
>,
now: DateTime.Utc,
): OrchestrationV2Run | undefined {
const run = projection.runs.reduce<OrchestrationV2Run | undefined>(
(latest, candidate) =>
candidate.status !== "queued" && (!latest || runRanAfter(candidate, latest))
? candidate
: latest,
undefined,
);
if (!run) return;
const preparedContinuation =
run.status === "starting" && run.restartContinuationOfRunId !== undefined;
const settled = run.status === "completed" || run.status === "waiting";
if (run.status !== "running" && !preparedContinuation && !settled) return;
if (restartContinuationBlocked(projection, run, now, settled)) return;
const liveTurnRequired = !preparedContinuation && !settled;
if (projection.thread.providerInstanceId !== run.providerInstanceId) return;
const providerThread = projection.providerThreads.find(
(thread) => thread.id === run.providerThreadId,
);
if (
!providerThread ||
providerThread.appThreadId !== projection.thread.id ||
providerThread.ownerNodeId !== null ||
providerThread.providerInstanceId !== run.providerInstanceId ||
providerThread.nativeThreadRef?.nativeId == null ||
providerThread.nativeThreadRef.strength !== "strong" ||
providerThread.nativeThreadRef.driver !== providerThread.driver ||
(liveTurnRequired && providerThread.status !== "active") ||
providerThread.status === "closed" ||
providerThread.status === "archived"
)
return;
const session = projection.providerSessions.find(
(candidate) => candidate.id === providerThread.providerSessionId,
);
// Most adapters keep a live session "ready" through its turns, so only a
// stopped or failed session rules out a live turn.
if (
session === undefined ||
session.providerInstanceId !== run.providerInstanceId ||
session.driver !== providerThread.driver ||
(!preparedContinuation && (session.status === "stopped" || session.status === "error"))
)
return;
if (settled) return hasOpenAwaitedWork(projection, providerThread) ? run : undefined;
if (
liveTurnRequired &&
!projection.providerTurns.some(
(turn) =>
turn.providerThreadId === providerThread.id &&
turn.runAttemptId === run.activeAttemptId &&
turn.status === "running",
)
)
return;
return run;
}
export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")(
function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) {
const settings = yield* ServerSettings.ServerSettingsService;
const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
if (!enabled) return;
const threads = yield* ThreadManagementService.ThreadManagementService;
const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);
const projection = yield* threads.getThreadRecords(
input.threadId,
["messages", "runs", "providerTurns", "attempts", "runtimeRequests", "turnItems"],
{ messageIds: [messageId], turnItemTypes: ["secret_request"] },
);
if (
!resolveProjectSettings(enabled, projection.thread.projectId).settings
.continueThreadsAfterServerUpdate
)
return;
if (projection.messages.some((message) => message.id === messageId)) return;
const source = projection.runs.find((run) => run.id === input.sourceRunId);
if (
!source ||
!isRestartContinuationSource(source, projection.providerTurns) ||
restartContinuationBlocked(
projection,
source,
yield* DateTime.now,
isRestartNoteSource(source, projection.providerTurns),
)
)
return;
if (
projection.runs.some(
(run) => run.id !== source.id && run.status !== "queued" && runRanAfter(run, source),
)
)
return;
if (projection.thread.providerInstanceId !== source.providerInstanceId) return;
// A turn that keeps crashing the server would otherwise restart it forever.
let chainLength = 0;
for (
let run: OrchestrationV2Run | undefined = source;
run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN;
run = projection.runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId)
) {
chainLength += 1;
}
if (chainLength >= MAX_CONTINUATION_CHAIN) {
yield* Effect.logWarning("Not continuing a run restarted too many times in a row", {
threadId: input.threadId,
sourceRunId: input.sourceRunId,
});
return;
}
const sourceRecords = yield* threads.getThreadRecords(
input.threadId,
["messages", "turnItems"],
{
messageIds: [source.userMessageId],
turnItemRunIds: [source.id],
turnItemTypes: ["run_interrupt_request"],
},
);
if (
sourceRecords.turnItems.some(
(item) => item.runId === source.id && item.type === "run_interrupt_request",
)
)
return;
const sourceMessage = sourceRecords.messages.find(
(message) => message.id === source.userMessageId,
);
if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return;
const note = restartContinuationNote(
source,
projection.runs,
projection.providerTurns,
projection.attempts,
);
const noteText =
note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work);
const text =
noteText === undefined
? CONTINUE_PROMPT
: note.settled
? noteText
: `${noteText}\n\n${CONTINUE_PROMPT}`;
yield* threads
.dispatch({
type: "message.dispatch",
commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`),
threadId: input.threadId,
messageId,
text,
notification: {
source: { kind: "system" },
outcome: "updated",
summary: note.settled
? "T3 Code restarted and stopped background work"
: "T3 Code restarted and resumed this turn",
detail: text,
},
attachments: [],
modelSelection: source.modelSelection,
dispatchMode: { type: "start_immediately" },
createdBy: "agent",
creationSource: "server",
restartContinuationOfRunId: input.sourceRunId,
})
.pipe(
// The project turned this instance off since the restart. No turn
// resumes, and the success path below still runs so a delegated
// child waiting on this run is recovered immediately instead of
// after EffectWorker exhausts its retries on a refusal that will
// not change.
Effect.catchTags({
ThreadManagementProviderInstanceDisabledError: (error) =>
Effect.logInfo("orchestration-v2.restart-continuation.instance-disabled", {
threadId: input.threadId,
sourceRunId: input.sourceRunId,
projectId: error.projectId,
instanceId: error.instanceId,
}),
}),
);
},
// A delegated child this declined to continue still owes its parent a
// result. Once a continuation run exists this is a no-op; that run settles it.
(effect, input) =>
effect.pipe(
Effect.andThen(
Effect.gen(function* () {
const threads = yield* ThreadManagementService.ThreadManagementService;
yield* threads.recoverDelegatedTask(input.threadId, input.sourceRunId);
}),
),
),
);