import * as Clock from "effect/Clock";
import * as Context from "effect/Context";
import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as RelayConfiguration from "../Config.ts";
import {
MANAGED_ENDPOINT_ZONE_OWNER_STAGE,
managedEndpointTunnelNamePrefix,
} from "../deploymentConfig.ts";
import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
export const MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES = 5;
// A tunnel that never connected is usually a link still being set up: a slow
// cloudflared download or a user who walked away mid-pairing. Give it an hour.
export const MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES = 60;
export const MANAGED_ENDPOINT_SWEEP_PAGE_SIZE = 100;
export const MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT = 100;
export const MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT = 10;
// Age buckets for legacy candidates, in days since the tunnel went down (or
// was created, for one that never connected).
const MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS = [7, 30, 90] as const;
// A host that never registered recovery cannot replace a deleted tunnel on
// its own build. Only delete its tunnel after it has been gone this long. A
// returning host that has updated recovers the tunnel at the same hostname;
// one that has not sees the client's "update T3 Code" message instead.
const MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS = 7;
// Deletions run a few at a time: each one is a row-locked database
// transaction plus two Cloudflare calls, so one at a time cannot finish a
// full attempt budget inside the cron's timeout. Each holds a Hyperdrive
// connection while it runs; keep this well under the 20-connection origin
// limit that request handlers share.
export const MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY = 4;
// Stop starting deletions this long after the sweep starts, leaving room under
// the cron's two-minute timeout to finish in-flight ones and record the
// counters. Counted from sweep start so slow listing eats into it.
const MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS = 90_000;
export interface ManagedEndpointSweepResult {
readonly mode: RelayConfiguration.ManagedEndpointCleanupMode;
readonly legacyMode: RelayConfiguration.ManagedEndpointCleanupMode;
readonly listRequests: number;
readonly scanned: number;
readonly attempted: number;
readonly deleted: number;
readonly wouldDelete: number;
/** Legacy tunnels past the legacy grace period, and how many were deleted. */
readonly wouldDeleteLegacy: number;
readonly deletedLegacy: number;
readonly skippedLegacy: number;
/** Legacy candidates, by days since they went down: over 7, 30, and 90. */
readonly legacyOver7Days: number;
readonly legacyOver30Days: number;
readonly legacyOver90Days: number;
readonly skippedOrphan: number;
/** The allocation row records a different tunnel under this name. */
readonly skippedReplaced: number;
/** The allocation row has not recorded a tunnel yet. */
readonly skippedUnrecorded: number;
/** Cloudflare's count of matching tunnels, before page limits. */
readonly totalDown: number | null;
readonly totalInactive: number | null;
readonly failed: number;
readonly truncated: boolean;
}
export class ManagedEndpointReaper extends Context.Service<
ManagedEndpointReaper,
{
readonly sweep: Effect.Effect<
ManagedEndpointSweepResult,
| ManagedEndpointProvider.ManagedEndpointTunnelClientError
| ManagedEndpointAllocations.ManagedEndpointAllocationPersistenceError
>;
}
>()("t3code-relay/environments/ManagedEndpointReaper") {}
function isExpiredManagedTunnel(input: {
readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel;
readonly status: "down" | "inactive";
readonly prefix: string;
readonly cutoff: DateTime.Utc;
}): input is typeof input & {
readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
readonly id: string;
readonly name: string;
};
} {
const { tunnel, status, prefix, cutoff } = input;
if (
typeof tunnel.id !== "string" ||
typeof tunnel.name !== "string" ||
tunnel.status !== status ||
!tunnel.name.startsWith(prefix) ||
!/^[a-f0-9]{16}$/u.test(tunnel.name.slice(prefix.length))
) {
return false;
}
const inactiveAt = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
if (typeof inactiveAt !== "string") {
return false;
}
const timestamp = DateTime.make(inactiveAt);
return Option.isSome(timestamp) && timestamp.value.epochMilliseconds <= cutoff.epochMilliseconds;
}
/** Days (fractional) since the tunnel went down, or was created if it never connected. */
function inactiveDaysAt(
tunnel: ManagedEndpointProvider.ManagedEndpointTunnel,
status: "down" | "inactive",
now: DateTime.Utc,
): number | null {
const since = status === "down" ? tunnel.connsInactiveAt : tunnel.createdAt;
if (typeof since !== "string") return null;
const timestamp = DateTime.make(since);
if (Option.isNone(timestamp)) return null;
return (now.epochMilliseconds - timestamp.value.epochMilliseconds) / 86_400_000;
}
function isRateLimited(cause: unknown): boolean {
if (typeof cause !== "object" || cause === null) {
return false;
}
if ("_tag" in cause && cause._tag === "TooManyRequests") {
return true;
}
if ("status" in cause && cause.status === 429) {
return true;
}
return "cause" in cause && isRateLimited(cause.cause);
}
function rotatedPages(input: {
readonly totalCount: number | undefined;
readonly slot: number;
readonly limit: number;
}): ReadonlyArray<number> {
if (input.limit <= 0) return [];
if (input.totalCount === undefined) {
return Array.from({ length: input.limit }, (_, index) => index + 2);
}
const laterPageCount = Math.max(
0,
Math.ceil(input.totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) - 1,
);
if (laterPageCount === 0) return [];
const count = Math.min(input.limit, laterPageCount);
const start = input.slot % laterPageCount;
return Array.from({ length: count }, (_, index) => 2 + ((start + index) % laterPageCount));
}
const emptyResult = (
mode: RelayConfiguration.ManagedEndpointCleanupMode,
legacyMode: RelayConfiguration.ManagedEndpointCleanupMode,
): ManagedEndpointSweepResult => ({
mode,
legacyMode,
listRequests: 0,
scanned: 0,
attempted: 0,
deleted: 0,
wouldDelete: 0,
wouldDeleteLegacy: 0,
deletedLegacy: 0,
skippedLegacy: 0,
legacyOver7Days: 0,
legacyOver30Days: 0,
legacyOver90Days: 0,
skippedOrphan: 0,
skippedReplaced: 0,
skippedUnrecorded: 0,
totalDown: null,
totalInactive: null,
failed: 0,
truncated: false,
});
export const make = Effect.gen(function* () {
const config = yield* RelayConfiguration.RelayConfiguration;
const tunnels = yield* ManagedEndpointProvider.ManagedEndpointTunnelClient;
const allocations = yield* ManagedEndpointAllocations.ManagedEndpointAllocations;
const provider = yield* ManagedEndpointProvider.ManagedEndpointProvider;
const sweep = Effect.gen(function* () {
const mode = config.managedEndpointCleanupMode ?? "off";
const legacyMode = config.legacyManagedEndpointCleanupMode ?? "off";
const namespace = config.managedEndpointNamespace;
if ((mode === "off" && legacyMode === "off") || !namespace) {
return emptyResult(mode, legacyMode);
}
const sweepStartedAtMillis = yield* Clock.currentTimeMillis;
// The override exists for the disposable canary stage; prod always
// waits the full grace period.
const legacyGraceMinutes =
namespace !== MANAGED_ENDPOINT_ZONE_OWNER_STAGE &&
config.legacyTunnelGraceMinutes !== undefined
? config.legacyTunnelGraceMinutes
: MANAGED_ENDPOINT_LEGACY_GRACE_PERIOD_DAYS * 24 * 60;
const legacyCutoff = DateTime.subtract(yield* DateTime.now, { minutes: legacyGraceMinutes });
const now = yield* DateTime.now;
const cutoffFor = (status: "down" | "inactive") =>
DateTime.subtract(now, {
minutes:
status === "down"
? MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES
: MANAGED_ENDPOINT_INACTIVE_GRACE_PERIOD_MINUTES,
});
const prefix = managedEndpointTunnelNamePrefix(namespace);
const slot = Math.floor(
now.epochMilliseconds / (MANAGED_ENDPOINT_GRACE_PERIOD_MINUTES * 60 * 1_000),
);
let listRequests = 0;
let truncated = false;
const totals: Record<"down" | "inactive", number | null> = { down: null, inactive: null };
const expired: Array<{
readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
readonly id: string;
readonly name: string;
};
readonly status: "down" | "inactive";
readonly cutoff: DateTime.Utc;
}> = [];
for (const status of ["down", "inactive"] as const) {
const cutoff = cutoffFor(status);
const cutoffIso = DateTime.formatIso(cutoff);
const listPage = (page: number) => {
listRequests += 1;
return tunnels.list({
isDeleted: false,
includePrefix: prefix,
status,
existedAt: cutoffIso,
...(status === "down" ? { wasInactiveAt: cutoffIso } : {}),
page,
perPage: MANAGED_ENDPOINT_SWEEP_PAGE_SIZE,
});
};
const first = yield* listPage(1);
const totalCount =
typeof first.resultInfo?.totalCount === "number" ? first.resultInfo.totalCount : undefined;
totals[status] = totalCount ?? null;
const pages = rotatedPages({
totalCount,
slot,
limit: Math.floor(MANAGED_ENDPOINT_SWEEP_LIST_REQUEST_LIMIT / 2) - 1,
});
const responses = [first, ...(yield* Effect.forEach(pages, listPage, { concurrency: 1 }))];
if (
totalCount !== undefined &&
Math.ceil(totalCount / MANAGED_ENDPOINT_SWEEP_PAGE_SIZE) > responses.length
) {
truncated = true;
} else if (
totalCount === undefined &&
responses.at(-1)?.result.length === MANAGED_ENDPOINT_SWEEP_PAGE_SIZE
) {
truncated = true;
}
for (const response of responses) {
expired.push(
...response.result
.map((tunnel) => ({ tunnel, status, prefix, cutoff }))
.filter(isExpiredManagedTunnel)
.map(({ tunnel }) => ({ tunnel, status, cutoff })),
);
}
}
const collected = [...new Map(expired.map((entry) => [entry.tunnel.id, entry])).values()];
// Start each sweep one attempt budget further along so a run of
// candidates whose deletes keep failing cannot hold the budget forever
// and starve everything listed after them.
const offset =
collected.length === 0 ? 0 : (slot * MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) % collected.length;
const uniqueExpired = [...collected.slice(offset), ...collected.slice(0, offset)];
const recorded = yield* allocations.listByTunnelNames(
uniqueExpired.map(({ tunnel }) => tunnel.name),
);
const recordedByTunnelName = new Map(
recorded.map((allocation) => [allocation.tunnelName, allocation]),
);
let attempted = 0;
let deleted = 0;
let wouldDelete = 0;
const candidates: Array<{
readonly owner: ManagedEndpointAllocations.ManagedEndpointTunnelAllocation;
readonly tunnel: ManagedEndpointProvider.ManagedEndpointTunnel & {
readonly id: string;
readonly name: string;
};
readonly status: "down" | "inactive";
readonly legacy: boolean;
readonly inactiveBefore: string;
}> = [];
let wouldDeleteLegacy = 0;
let deletedLegacy = 0;
let skippedLegacy = 0;
const legacyOverDays = new Map<number, number>(
MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS.map((days) => [days, 0]),
);
let skippedOrphan = 0;
let skippedReplaced = 0;
let skippedUnrecorded = 0;
let failed = 0;
for (const { tunnel, status, cutoff } of uniqueExpired) {
const allocation = recordedByTunnelName.get(tunnel.name);
if (
allocation !== undefined &&
allocation.tunnelId !== null &&
allocation.tunnelId !== tunnel.id
) {
skippedReplaced += 1;
continue;
}
const owner = allocation?.tunnelId === tunnel.id ? allocation : undefined;
if (allocation !== undefined && owner === undefined) {
skippedUnrecorded += 1;
continue;
}
// A tunnel with no allocation row cannot be claimed, so a relink that
// adopts it by name races any delete here. Count it and leave it for a
// manual sweep instead.
if (owner === undefined) {
skippedOrphan += 1;
continue;
}
const legacy = !owner.recoveryEnabled;
if (legacy) {
skippedLegacy += 1;
const inactiveDays = inactiveDaysAt(tunnel, status, now);
for (const days of MANAGED_ENDPOINT_LEGACY_AGE_BUCKET_DAYS) {
if (inactiveDays !== null && inactiveDays > days) {
legacyOverDays.set(days, (legacyOverDays.get(days) ?? 0) + 1);
}
}
if (
legacyMode === "off" ||
!isExpiredManagedTunnel({ tunnel, status, prefix, cutoff: legacyCutoff })
) {
continue;
}
wouldDeleteLegacy += 1;
if (legacyMode === "dry-run") continue;
} else {
if (mode === "off") continue;
wouldDelete += 1;
if (mode === "dry-run") continue;
}
if (candidates.length >= MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT) {
truncated = true;
break;
}
candidates.push({
owner,
tunnel,
status,
legacy,
// The release re-reads the tunnel and deletes only if it is still in
// this status and inactive since before this cutoff.
inactiveBefore: DateTime.formatIso(legacy ? legacyCutoff : cutoff),
});
}
const deleteDeadline = sweepStartedAtMillis + MANAGED_ENDPOINT_SWEEP_DELETE_BUDGET_MS;
let stopDeleting = false;
yield* Effect.forEach(
candidates,
(candidate) =>
Effect.gen(function* () {
if (stopDeleting || (yield* Clock.currentTimeMillis) >= deleteDeadline) {
stopDeleting = true;
truncated = true;
return;
}
attempted += 1;
const result = yield* provider
.release({
userId: candidate.owner.userId,
environmentId: candidate.owner.environmentId,
expectedTunnelId: candidate.tunnel.id,
expectedInactiveBefore: candidate.inactiveBefore,
expectedStatus: candidate.status,
// Only a legacy host needs an update to recover; tell its user so.
...(candidate.legacy ? { markReleased: true } : {}),
})
.pipe(Effect.result);
if (result._tag === "Failure") {
failed += 1;
yield* Effect.logWarning("Failed to delete an inactive managed tunnel", {
tunnelId: candidate.tunnel.id,
tunnelName: candidate.tunnel.name,
legacy: candidate.legacy,
cause: result.failure,
});
if (isRateLimited(result.failure)) {
stopDeleting = true;
truncated = true;
}
} else if (result.success) {
deleted += 1;
if (candidate.legacy) deletedLegacy += 1;
yield* Effect.logInfo("Deleted an inactive managed tunnel", {
tunnelId: candidate.tunnel.id,
tunnelName: candidate.tunnel.name,
status: candidate.status,
legacy: candidate.legacy,
});
}
}),
{ concurrency: MANAGED_ENDPOINT_SWEEP_DELETE_CONCURRENCY, discard: true },
);
return {
mode,
legacyMode,
listRequests,
scanned: uniqueExpired.length,
attempted,
deleted,
wouldDelete,
wouldDeleteLegacy,
deletedLegacy,
skippedLegacy,
legacyOver7Days: legacyOverDays.get(7) ?? 0,
legacyOver30Days: legacyOverDays.get(30) ?? 0,
legacyOver90Days: legacyOverDays.get(90) ?? 0,
skippedOrphan,
skippedReplaced,
skippedUnrecorded,
totalDown: totals.down,
totalInactive: totals.inactive,
failed,
truncated,
};
}).pipe(
// Dry-run rollout reads these counters from the exported span.
Effect.tap((result) =>
Effect.annotateCurrentSpan(
Object.fromEntries(
Object.entries(result).map(([key, value]) => [
`relay.managed_endpoint_reaper.${key}`,
value,
]),
),
),
),
Effect.withSpan("relay.managed_endpoint_reaper.sweep"),
);
return ManagedEndpointReaper.of({ sweep });
});
export const layer = Layer.effect(ManagedEndpointReaper, make);