import * as NodeCrypto from "node:crypto";
import {
AuthRelayReadScope,
AuthRelayWriteScope,
AuthStandardClientScopes,
EnvironmentCloudEndpointUnavailableError,
EnvironmentCloudLinkStateResult,
EnvironmentCloudRelayConfigResult,
EnvironmentHttpApi,
EnvironmentHttpBadRequestError,
EnvironmentHttpConflictError,
EnvironmentHttpInternalServerError,
EnvironmentHttpUnauthorizedError,
} from "@t3tools/contracts";
import {
RelayCloudEnvironmentHealthProofPayload,
RelayCloudEnvironmentHealthRequest,
RelayCloudMintCredentialProofPayload,
RelayCloudMintCredentialRequest,
RelayEnvironmentHealthResponseProofPayload,
type RelayEnvironmentHealthResponse as RelayEnvironmentHealthResponseShape,
RelayEnvironmentConfigRequest,
RelayEnvironmentLinkChallengeResponse,
RelayEnvironmentLinkResponse,
RelayEnvironmentMintResponseProofPayload,
type RelayEnvironmentMintResponse as RelayEnvironmentMintResponseShape,
RelayEnvironmentLinkProof,
RelayEnvironmentLinkProofPayload,
RelayLinkProofRequest,
RelayManagedEndpointOrigin,
RelayOkResponse,
} from "@t3tools/contracts/relay";
import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
import {
normalizeRelayIssuer,
RELAY_HEALTH_REQUEST_TYP,
RELAY_HEALTH_RESPONSE_TYP,
RELAY_LINK_PROOF_TYP,
RELAY_MINT_REQUEST_TYP,
RELAY_MINT_RESPONSE_TYP,
signRelayJwt,
verifyRelayJwt,
} from "@t3tools/shared/relayJwt";
import { isSecureRelayUrl } from "@t3tools/shared/relayUrl";
import * as DateTime from "effect/DateTime";
import * as Crypto from "effect/Crypto";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Schema from "effect/Schema";
import * as HttpEffect from "effect/unstable/http/HttpEffect";
import { HttpServerRequest, HttpServerResponse } from "effect/unstable/http";
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
import * as EnvironmentAuth from "../auth/EnvironmentAuth.ts";
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
import { requireEnvironmentScope } from "../auth/http.ts";
import * as ServerConfig from "../config.ts";
import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
import {
SERVICE_STATE_FILE,
SERVICE_STOP_MARKER_FILE,
serviceStateHasPendingUpdate,
} from "./serviceProtocol.ts";
import {
CLOUD_ENDPOINT_RUNTIME_CONFIG,
CLOUD_LINKED_USER_ID,
CLOUD_MINT_PUBLIC_KEY,
encodeEndpointRuntimeConfigJson,
PUBLISH_AGENT_ACTIVITY_SECRET,
RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
RELAY_ISSUER_SECRET,
RELAY_URL_SECRET,
} from "./config.ts";
import { relayUrlConfig } from "./publicConfig.ts";
import {
readCliDesiredCloudLink,
readCliDesiredLinkMode,
setCliDesiredCloudLink,
} from "./CliState.ts";
import * as CliTokenManager from "./CliTokenManager.ts";
import { getOrCreateEnvironmentKeyPairFromSecretStore } from "./environmentKeys.ts";
import { traceRelayRequest } from "./traceRelayRequest.ts";
import { filterRelayResponse, relayRequestError } from "./relayResponse.ts";
const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
const CLOUD_CREDENTIAL_RESPONSE_HEADERS = {
"cache-control": "no-store",
pragma: "no-cache",
} as const;
const appendCloudCredentialResponseHeaders = HttpEffect.appendPreResponseHandler(
(_request, response) =>
Effect.succeed(HttpServerResponse.setHeaders(response, CLOUD_CREDENTIAL_RESPONSE_HEADERS)),
);
const failEnvironmentCloudInternalError =
(message: string) =>
(cause: unknown): Effect.Effect<never, EnvironmentHttpInternalServerError> =>
Effect.logError(message, { cause }).pipe(
Effect.flatMap(() => Effect.fail(new EnvironmentHttpInternalServerError({ message }))),
);
const failCloudCliTokenManagerError = (error: CliTokenManager.CloudCliTokenManagerError) =>
failEnvironmentCloudInternalError(error.message)(error);
const requireRelayUrl = relayUrlConfig.pipe(
Effect.mapError(
() =>
new EnvironmentHttpInternalServerError({
message: "T3CODE_RELAY_URL must be configured as a secure absolute HTTPS origin.",
}),
),
);
function bytesToString(bytes: Uint8Array): string {
return new TextDecoder().decode(bytes);
}
function stringToBytes(value: string): Uint8Array {
return new TextEncoder().encode(value);
}
export function consumeCloudReplayGuards(input: {
readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
readonly names: ReadonlyArray<string>;
readonly value: Uint8Array;
}) {
return Effect.all(
input.names.map((name) =>
input.secrets.create(name, input.value).pipe(
Effect.as(true),
Effect.catchIf(ServerSecretStore.isSecretStoreError, (error) =>
ServerSecretStore.isSecretAlreadyExistsError(error)
? Effect.succeed(false)
: Effect.fail(error),
),
),
),
{ concurrency: input.names.length },
).pipe(Effect.map((created) => created.every(Boolean)));
}
function normalizePemForSignedPayload(value: string): string {
return value.trim();
}
function normalizeHostname(hostname: string): string {
return hostname
.trim()
.toLowerCase()
.replace(/^\[(.*)\]$/, "$1");
}
function validateCloudMintPublicKey(
publicKey: string,
): Effect.Effect<void, EnvironmentHttpBadRequestError> {
return Effect.try({
try: () => NodeCrypto.createPublicKey(publicKey.replace(/\\n/g, "\n")),
catch: () =>
new EnvironmentHttpBadRequestError({
message: "Cloud mint public key must be a valid Ed25519 public key.",
}),
}).pipe(
Effect.flatMap((key) =>
key.asymmetricKeyType === "ed25519"
? Effect.void
: Effect.fail(
new EnvironmentHttpBadRequestError({
message: "Cloud mint public key must be a valid Ed25519 public key.",
}),
),
),
);
}
function validateRelayConfigPayload(
payload: RelayEnvironmentConfigRequest,
): Effect.Effect<void, EnvironmentHttpBadRequestError> {
if (!isSecureRelayUrl(payload.relayUrl)) {
return Effect.fail(
new EnvironmentHttpBadRequestError({
message: "Relay URL must be a secure absolute HTTPS URL.",
}),
);
}
if (payload.relayIssuer !== undefined && !isSecureRelayUrl(payload.relayIssuer)) {
return Effect.fail(
new EnvironmentHttpBadRequestError({
message: "Relay issuer must be a secure absolute HTTPS URL.",
}),
);
}
if (payload.environmentCredential.trim().length === 0) {
return Effect.fail(
new EnvironmentHttpBadRequestError({
message: "Relay environment credential is required.",
}),
);
}
if (payload.cloudUserId.trim().length === 0) {
return Effect.fail(
new EnvironmentHttpBadRequestError({
message: "Cloud user id is required.",
}),
);
}
return Effect.void;
}
function validateLinkedCloudUser(input: {
readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
readonly cloudUserId: string;
}): Effect.Effect<void, EnvironmentAuth.ServerAuthInternalError | EnvironmentHttpConflictError> {
return input.secrets.get(CLOUD_LINKED_USER_ID).pipe(
Effect.mapError(
(cause) =>
new EnvironmentAuth.ServerAuthLinkedCloudAccountVerificationError({
cause,
}),
),
Effect.flatMap((existing) => {
if (Option.isNone(existing)) {
return Effect.void;
}
const existingCloudUserId = bytesToString(existing.value);
return existingCloudUserId === input.cloudUserId
? Effect.void
: Effect.fail(
new EnvironmentHttpConflictError({
message:
"This environment is already linked to a different cloud account. Unlink it before switching accounts.",
}),
);
}),
);
}
function readInstalledCloudUserId(
secrets: ServerSecretStore.ServerSecretStore["Service"],
): Effect.Effect<string, EnvironmentAuth.ServerAuthInternalError> {
return secrets.get(CLOUD_LINKED_USER_ID).pipe(
Effect.mapError(
(cause) =>
new EnvironmentAuth.ServerAuthLinkedCloudAccountReadError({
cause,
}),
),
Effect.flatMap((bytes) =>
Option.isSome(bytes)
? Effect.succeed(bytesToString(bytes.value))
: Effect.fail(new EnvironmentAuth.ServerAuthLinkedCloudAccountMissingError({})),
),
);
}
function isLoopbackHostname(hostname: string): boolean {
return LOOPBACK_HOSTNAMES.has(normalizeHostname(hostname));
}
function firstForwardedHeaderValue(value: string | undefined): string | undefined {
const first = value?.split(",")[0]?.trim();
return first && first.length > 0 ? first : undefined;
}
function requestAbsoluteUrl(request: HttpServerRequest.HttpServerRequest): string | null {
try {
return new URL(request.originalUrl).href;
} catch {
const host = firstForwardedHeaderValue(request.headers.host) ?? "127.0.0.1";
try {
return new URL(request.originalUrl, `http://${host}`).href;
} catch {
return null;
}
}
}
function hasForwardedAuthorityHeaders(request: HttpServerRequest.HttpServerRequest): boolean {
return (
firstForwardedHeaderValue(request.headers["x-forwarded-host"]) !== undefined ||
firstForwardedHeaderValue(request.headers["x-forwarded-proto"]) !== undefined
);
}
function endpointRequestPort(url: URL): number {
return Number(url.port || (url.protocol === "https:" ? 443 : 80));
}
function isAllowedEndpointOrigin(input: {
readonly origin: RelayManagedEndpointOrigin;
readonly requestUrl: string;
}): boolean {
if (!isLoopbackHostname(input.origin.localHttpHost)) {
return false;
}
const url = new URL(input.requestUrl);
if (!isLoopbackHostname(url.hostname)) {
return false;
}
return input.origin.localHttpPort === endpointRequestPort(url);
}
// A managed (Cloudflare tunnel) endpoint is provisioned by the relay and must
// point at a loopback origin. A manual endpoint is reached out of band (e.g.
// Tailscale) or not advertised at all for publish-only links, so it is not
// tied to the managed-tunnel scope.
export function isSupportedLinkProviderKind(request: RelayLinkProofRequest): boolean {
return (
request.endpoint.providerKind === "cloudflare_tunnel" ||
request.endpoint.providerKind === "manual"
);
}
export function linkProofScopes(
request: RelayLinkProofRequest,
): RelayEnvironmentLinkProofPayload["scopes"] {
return request.endpoint.providerKind === "cloudflare_tunnel"
? ["agent_activity_notifications", "managed_tunnels"]
: ["agent_activity_notifications"];
}
function hasExactScope(input: {
readonly scopes: ReadonlyArray<string>;
readonly expected: string;
}): boolean {
return input.scopes.length === 1 && input.scopes[0] === input.expected;
}
function hasBoundedCloudProofLifetime(input: {
readonly iat: number;
readonly exp: number;
readonly nowSeconds: number;
}): boolean {
return (
input.exp > input.iat &&
input.exp - input.iat <= CLOUD_PROOF_MAX_LIFETIME_SECONDS &&
input.iat <= input.nowSeconds + CLOUD_PROOF_CLOCK_SKEW_SECONDS
);
}
const decodeCloudHealthProof = Schema.decodeUnknownEffect(RelayCloudEnvironmentHealthProofPayload);
const decodeCloudMintProof = Schema.decodeUnknownEffect(RelayCloudMintCredentialProofPayload);
interface CloudHttpDependencies {
readonly secrets: ServerSecretStore.ServerSecretStore["Service"];
readonly environment: ServerEnvironment.ServerEnvironment["Service"];
readonly endpointRuntime: ManagedEndpointRuntime.CloudManagedEndpointRuntime["Service"];
readonly environmentAuth: EnvironmentAuth.EnvironmentAuth["Service"];
readonly cliTokenManager: CliTokenManager.CloudCliTokenManager["Service"];
readonly httpClient: HttpClient.HttpClient;
}
const cloudHttpDependencies = Effect.gen(function* () {
return {
secrets: yield* ServerSecretStore.ServerSecretStore,
environment: yield* ServerEnvironment.ServerEnvironment,
endpointRuntime: yield* ManagedEndpointRuntime.CloudManagedEndpointRuntime,
environmentAuth: yield* EnvironmentAuth.EnvironmentAuth,
cliTokenManager: yield* CliTokenManager.CloudCliTokenManager,
httpClient: yield* HttpClient.HttpClient,
} satisfies CloudHttpDependencies;
});
const makeCloudLinkProof = Effect.fn("environment.cloud.makeLinkProof")(function* (
dependencies: CloudHttpDependencies,
request: RelayLinkProofRequest,
requestUrl: string,
) {
const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
if (
!isSupportedLinkProviderKind(request) ||
!isAllowedEndpointOrigin({
origin: request.origin,
requestUrl,
})
) {
return yield* new EnvironmentHttpBadRequestError({
message: "Invalid managed endpoint origin.",
});
}
const now = yield* DateTime.now;
const expiresAt = DateTime.add(now, { minutes: 5 });
const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
const descriptor = yield* dependencies.environment.getDescriptor;
const payload = {
iss: `t3-env:${descriptor.environmentId}`,
aud: normalizeRelayIssuer(request.relayIssuer),
sub: descriptor.environmentId,
jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
iat: nowSeconds,
exp: Math.floor(expiresAt.epochMilliseconds / 1_000),
challenge: request.challenge,
descriptor,
environmentId: descriptor.environmentId,
environmentPublicKey: normalizePemForSignedPayload(keyPair.publicKey),
endpoint: request.endpoint,
origin: request.origin,
scopes: linkProofScopes(request),
} satisfies RelayEnvironmentLinkProofPayload;
return yield* signRelayJwt({
privateKey: keyPair.privateKey,
typ: RELAY_LINK_PROOF_TYP,
payload,
}).pipe(
Effect.mapError(
(cause) =>
new EnvironmentAuth.ServerAuthCloudLinkJwtSigningError({
cause,
}),
),
);
});
const cloudLinkProofHandler = Effect.fn("environment.cloud.linkProof")(
function* (dependencies: CloudHttpDependencies, request: RelayLinkProofRequest) {
yield* requireEnvironmentScope(AuthRelayWriteScope);
const httpRequest = yield* HttpServerRequest.HttpServerRequest;
const requestUrl = requestAbsoluteUrl(httpRequest);
if (requestUrl === null || hasForwardedAuthorityHeaders(httpRequest)) {
return yield* new EnvironmentHttpBadRequestError({
message: "Invalid managed endpoint origin.",
});
}
const proof = yield* makeCloudLinkProof(dependencies, request, requestUrl);
yield* appendCloudCredentialResponseHeaders;
return proof satisfies RelayEnvironmentLinkProof;
},
Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
failEnvironmentCloudInternalError(error.message)(error),
),
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not generate environment link proof."),
),
Effect.catchTag(
"PlatformError",
failEnvironmentCloudInternalError("Could not generate environment link proof."),
),
);
const applyCloudRelayConfig = Effect.fn("environment.cloud.applyRelayConfig")(function* (
dependencies: CloudHttpDependencies,
payload: RelayEnvironmentConfigRequest,
) {
yield* validateRelayConfigPayload(payload);
yield* validateLinkedCloudUser({
secrets: dependencies.secrets,
cloudUserId: payload.cloudUserId,
});
yield* validateCloudMintPublicKey(payload.cloudMintPublicKey);
const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(
payload.endpointRuntime,
);
const ok =
endpointRuntimeStatus.status === "disabled" || endpointRuntimeStatus.status === "running";
if (!ok) {
return yield* new EnvironmentCloudEndpointUnavailableError({
message: "Managed endpoint runtime could not be started.",
endpointRuntimeStatus,
});
}
yield* dependencies.secrets.set(RELAY_URL_SECRET, stringToBytes(payload.relayUrl));
yield* dependencies.secrets.set(
RELAY_ISSUER_SECRET,
stringToBytes(payload.relayIssuer ?? payload.relayUrl),
);
yield* dependencies.secrets.set(CLOUD_LINKED_USER_ID, stringToBytes(payload.cloudUserId));
yield* dependencies.secrets.set(
RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
stringToBytes(payload.environmentCredential),
);
yield* dependencies.secrets.set(CLOUD_MINT_PUBLIC_KEY, stringToBytes(payload.cloudMintPublicKey));
if (payload.endpointRuntime) {
const endpointRuntimeJson = yield* encodeEndpointRuntimeConfigJson(payload.endpointRuntime);
yield* dependencies.secrets.set(
CLOUD_ENDPOINT_RUNTIME_CONFIG,
stringToBytes(endpointRuntimeJson),
);
} else {
yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
}
return { ok, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
});
const cloudRelayConfigHandler = Effect.fn("environment.cloud.relayConfig")(
function* (dependencies: CloudHttpDependencies, payload: RelayEnvironmentConfigRequest) {
yield* requireEnvironmentScope(AuthRelayWriteScope);
return yield* applyCloudRelayConfig(dependencies, payload);
},
Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
failEnvironmentCloudInternalError(error.message)(error),
),
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
),
Effect.catchTag(
"SchemaError",
failEnvironmentCloudInternalError("Could not persist environment relay configuration."),
),
);
const relayClientRequest = <A>(
dependencies: CloudHttpDependencies,
input: {
readonly url: string;
readonly token: string;
readonly payload: unknown;
readonly schema: Schema.Decoder<A>;
},
) =>
HttpClientRequest.post(input.url).pipe(
HttpClientRequest.bearerToken(input.token),
HttpClientRequest.bodyJson(input.payload),
Effect.flatMap(dependencies.httpClient.execute),
Effect.flatMap(filterRelayResponse),
Effect.flatMap(HttpClientResponse.schemaBodyJson(input.schema)),
Effect.mapError(relayRequestError),
withRelayClientTracing,
);
const reconcileDesiredCloudLinkWith = Effect.fn("environment.cloud.reconcileDesiredLinkWith")(
function* (dependencies: CloudHttpDependencies, localOrigin: string) {
const localUrl = yield* Effect.try({
try: () => new URL(localOrigin),
catch: () =>
new EnvironmentHttpBadRequestError({
message: "Could not resolve local environment origin.",
}),
});
if (localUrl.origin !== localOrigin) {
return yield* new EnvironmentHttpBadRequestError({
message: "Could not resolve local environment origin.",
});
}
const localWsOrigin = localOrigin.replace(/^http/u, "ws");
const token = yield* dependencies.cliTokenManager.getExisting.pipe(
Effect.flatMap(
Option.match({
onNone: () =>
Effect.fail(
new EnvironmentHttpUnauthorizedError({
message: "Run `t3 connect link` to authorize this environment.",
}),
),
onSome: Effect.succeed,
}),
),
);
const mode = yield* readCliDesiredLinkMode;
const managedTunnelsEnabled = mode !== "publish_only";
const relayUrl = yield* requireRelayUrl;
const challenge = yield* relayClientRequest(dependencies, {
url: `${relayUrl}/v1/client/environment-link-challenges`,
token: token.accessToken,
payload: {
notificationsEnabled: true,
liveActivitiesEnabled: true,
managedTunnelsEnabled,
},
schema: RelayEnvironmentLinkChallengeResponse,
});
const proof = yield* makeCloudLinkProof(
dependencies,
{
challenge: challenge.challenge,
relayIssuer: relayUrl,
endpoint: {
httpBaseUrl: localOrigin,
wsBaseUrl: localWsOrigin,
providerKind: managedTunnelsEnabled ? "cloudflare_tunnel" : "manual",
},
origin: {
localHttpHost: localUrl.hostname,
localHttpPort: endpointRequestPort(localUrl),
},
},
localOrigin,
);
const link = yield* relayClientRequest(dependencies, {
url: `${relayUrl}/v1/client/environment-links`,
token: token.accessToken,
payload: {
proof,
notificationsEnabled: true,
liveActivitiesEnabled: true,
managedTunnelsEnabled,
},
schema: RelayEnvironmentLinkResponse,
});
yield* setCliDesiredCloudLink(true, mode);
return yield* applyCloudRelayConfig(dependencies, {
relayUrl,
relayIssuer: link.relayIssuer,
cloudUserId: link.cloudUserId,
environmentCredential: link.environmentCredential,
cloudMintPublicKey: link.cloudMintPublicKey,
endpointRuntime: link.endpointRuntime,
});
},
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not persist desired T3 Connect link state."),
),
Effect.catchTags({
CloudCliCredentialRemovalError: failCloudCliTokenManagerError,
CloudCliCredentialRefreshError: failCloudCliTokenManagerError,
CloudCliCredentialReadError: failCloudCliTokenManagerError,
CloudCliAuthorizationError: failCloudCliTokenManagerError,
CloudCliAuthorizationTimeoutError: failCloudCliTokenManagerError,
}),
);
export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileDesiredLink")(
function* (localOrigin: string) {
return yield* reconcileDesiredCloudLinkWith(yield* cloudHttpDependencies, localOrigin);
},
);
// The launcher owns this durable state, so read it directly both when a trial
// decides whether it owns pre-activation cleanup and while a server tears down.
export const pendingServiceUpdateExists = Effect.gen(function* () {
const config = yield* ServerConfig.ServerConfig;
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const runtimeDir = path.join(config.baseDir, "runtime");
const stateText = yield* fs
.readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
.pipe(Effect.option);
return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
});
// A pending update alone is not proof a replacement server is coming: an
// explicit launcher stop (`t3 service uninstall`, `systemctl stop`,
// `launchctl bootout`) during
// the pending window also tears this server down. The launcher marks that case
// just before it signals the child, so pending + no marker is the handoff.
const pendingUpdateHandoffExists = Effect.gen(function* () {
if (!(yield* pendingServiceUpdateExists)) {
return false;
}
const config = yield* ServerConfig.ServerConfig;
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
const runtimeDir = path.join(config.baseDir, "runtime");
const stopping = yield* fs
.exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
.pipe(Effect.orElseSucceed(() => false));
return !stopping;
});
// Cloudflare bills per provisioned tunnel, so an environment that goes offline
// must not leave its tunnel behind. Releasing deletes only the tunnel — the
// relay keeps the link and its hostname reservation, and the next startup's
// link reconcile provisions a replacement tunnel under the same URL.
export const releaseManagedTunnelOnShutdown = Effect.fn(
"environment.cloud.releaseManagedTunnelOnShutdown",
)(function* () {
const dependencies = yield* cloudHttpDependencies;
// Only a managed link stores a runtime config; publish-only links have no
// tunnel to release.
const runtimeConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
if (Option.isNone(runtimeConfig)) {
return false;
}
// Only CLI-desired managed links release on shutdown, because the startup
// reconcile that provisions the replacement tunnel only runs for them. A
// link installed by a web/mobile client comes back after a restart by
// reapplying the stored connector token — it has no boot-time re-provision
// path — so its tunnel must survive the restart. (Unlink still deletes it.)
if (!(yield* readCliDesiredCloudLink) || (yield* readCliDesiredLinkMode) !== "managed") {
return false;
}
// A shutdown that hands off to a pending remote update is not the
// environment going offline: the launcher immediately brings a server back
// (the new version, or the old one after a rollback). Deleting the tunnel
// here forces that server to provision a replacement UUID, and the public
// hostname's route to the new tunnel takes 1-2 minutes to propagate — the
// dominant cost of an update restart. Keep the tunnel instead: the next
// boot respawns the connector from the stored config and is reachable as
// soon as it connects, and the reconcile confirms the still-live tunnel
// without replacing it.
if (yield* pendingUpdateHandoffExists) {
yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
return false;
}
const token = yield* dependencies.cliTokenManager.getExisting;
if (Option.isNone(token)) {
return false;
}
// The link belongs to the relay it was installed against, so target the
// persisted URL: T3CODE_RELAY_URL may have changed since the link was made.
const relayUrl = yield* dependencies.secrets.get(RELAY_URL_SECRET);
if (Option.isNone(relayUrl)) {
return false;
}
const environmentId = yield* dependencies.environment.getEnvironmentId;
// Stop the local connector before the relay deletes the tunnel it serves.
yield* dependencies.endpointRuntime.applyConfig(null);
const response = yield* HttpClientRequest.delete(
`${bytesToString(relayUrl.value)}/v1/client/environment-links/${encodeURIComponent(environmentId)}/tunnel`,
).pipe(
HttpClientRequest.bearerToken(token.value.accessToken),
dependencies.httpClient.execute,
Effect.flatMap(filterRelayResponse),
Effect.flatMap(HttpClientResponse.schemaBodyJson(RelayOkResponse)),
withRelayClientTracing,
);
// ok:false means the relay skipped deletion because a concurrent provision
// owns the recorded tunnel now — leave the stored config alone.
if (!response.ok) {
return false;
}
// The connector token died with the tunnel. Drop the stored config so the
// next start waits for the link reconcile instead of respawning the relay
// client with a dead token. Kept when the release request fails: the tunnel
// still exists, so the stored token keeps working across the restart.
// Only dropped while it is still the config this shutdown released — a fast
// restart may already have reconciled and stored a fresh config for its
// replacement tunnel, and that one must survive this finalizer.
const storedConfig = yield* dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG);
if (
Option.isSome(storedConfig) &&
bytesToString(storedConfig.value) === bytesToString(runtimeConfig.value)
) {
yield* dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG);
}
return true;
});
const readCloudLinkState = Effect.fn("environment.cloud.readLinkState")(function* (
dependencies: CloudHttpDependencies,
) {
const [cloudUserId, relayUrl, relayIssuer, endpointRuntimeConfig, publishAgentActivity] =
yield* Effect.all(
[
dependencies.secrets.get(CLOUD_LINKED_USER_ID),
dependencies.secrets.get(RELAY_URL_SECRET),
dependencies.secrets.get(RELAY_ISSUER_SECRET),
dependencies.secrets.get(CLOUD_ENDPOINT_RUNTIME_CONFIG),
dependencies.secrets.get(PUBLISH_AGENT_ACTIVITY_SECRET),
],
{ concurrency: 5 },
);
return {
linked: Option.isSome(cloudUserId),
cloudUserId: Option.isSome(cloudUserId) ? bytesToString(cloudUserId.value) : null,
relayUrl: Option.isSome(relayUrl) ? bytesToString(relayUrl.value) : null,
relayIssuer: Option.isSome(relayIssuer) ? bytesToString(relayIssuer.value) : null,
// The managed tunnel runtime config is only stored for managed links; a
// publish-only link leaves it absent.
managedTunnelActive: Option.isSome(endpointRuntimeConfig),
publishAgentActivity: Option.isSome(publishAgentActivity)
? bytesToString(publishAgentActivity.value) === "true"
: false,
} satisfies EnvironmentCloudLinkStateResult;
});
const cloudLinkStateHandler = Effect.fn("environment.cloud.linkState")(
function* (dependencies: CloudHttpDependencies) {
yield* requireEnvironmentScope(AuthRelayReadScope);
return yield* readCloudLinkState(dependencies);
},
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not read environment relay configuration."),
),
);
const cloudUnlinkHandler = Effect.fn("environment.cloud.unlink")(
function* (dependencies: CloudHttpDependencies) {
yield* requireEnvironmentScope(AuthRelayWriteScope);
const endpointRuntimeStatus = yield* dependencies.endpointRuntime.applyConfig(null);
yield* Effect.all(
[
dependencies.secrets.remove(CLOUD_LINKED_USER_ID),
dependencies.secrets.remove(RELAY_URL_SECRET),
dependencies.secrets.remove(RELAY_ISSUER_SECRET),
dependencies.secrets.remove(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
dependencies.secrets.remove(CLOUD_MINT_PUBLIC_KEY),
dependencies.secrets.remove(CLOUD_ENDPOINT_RUNTIME_CONFIG),
dependencies.secrets.remove(PUBLISH_AGENT_ACTIVITY_SECRET),
],
{ concurrency: 7 },
);
yield* setCliDesiredCloudLink(false);
return { ok: true, endpointRuntimeStatus } satisfies EnvironmentCloudRelayConfigResult;
},
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not remove environment relay configuration."),
),
);
const cloudPreferencesHandler = Effect.fn("environment.cloud.preferences")(
function* (
dependencies: CloudHttpDependencies,
payload: { readonly publishAgentActivity: boolean },
) {
yield* requireEnvironmentScope(AuthRelayWriteScope);
yield* dependencies.secrets.set(
PUBLISH_AGENT_ACTIVITY_SECRET,
stringToBytes(String(payload.publishAgentActivity)),
);
return yield* readCloudLinkState(dependencies);
},
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not persist environment cloud preferences."),
),
);
const cloudEnvironmentHealthHandler = Effect.fn("environment.cloud.health")(
function* (dependencies: CloudHttpDependencies, request: RelayCloudEnvironmentHealthRequest) {
const cloudMintPublicKey = yield* dependencies.secrets
.get(CLOUD_MINT_PUBLIC_KEY)
.pipe(
Effect.flatMap((bytes) =>
Option.isSome(bytes)
? Effect.succeed(bytesToString(bytes.value))
: Effect.fail(new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({})),
),
);
const relayIssuer = yield* dependencies.secrets
.get(RELAY_ISSUER_SECRET)
.pipe(
Effect.flatMap((bytes) =>
Option.isSome(bytes)
? Effect.succeed(bytesToString(bytes.value))
: dependencies.secrets
.get(RELAY_URL_SECRET)
.pipe(
Effect.flatMap((fallbackBytes) =>
Option.isSome(fallbackBytes)
? Effect.succeed(bytesToString(fallbackBytes.value))
: Effect.fail(new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({})),
),
),
),
);
const environmentId = yield* dependencies.environment.getEnvironmentId;
const linkedCloudUserId = yield* readInstalledCloudUserId(dependencies.secrets);
const now = yield* DateTime.now;
const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
const proofOption = yield* verifyRelayJwt({
publicKey: cloudMintPublicKey,
token: request.proof,
typ: RELAY_HEALTH_REQUEST_TYP,
issuer: normalizeRelayIssuer(relayIssuer),
audience: `t3-env:${environmentId}`,
nowEpochSeconds: nowSeconds,
}).pipe(Effect.flatMap(decodeCloudHealthProof), Effect.option);
if (
Option.isNone(proofOption) ||
proofOption.value.environmentId !== environmentId ||
proofOption.value.sub !== linkedCloudUserId ||
!hasBoundedCloudProofLifetime({ ...proofOption.value, nowSeconds }) ||
!hasExactScope({ scopes: proofOption.value.scope, expected: "environment:status" })
) {
return yield* new EnvironmentHttpUnauthorizedError({
message: "Invalid cloud health request.",
});
}
const proof = proofOption.value;
const jtiSecretName = `${CLOUD_HEALTH_JTI_PREFIX}${proof.jti}`;
const nonceSecretName = `${CLOUD_HEALTH_NONCE_PREFIX}${proof.nonce}`;
const consumedReplayGuards = yield* consumeCloudReplayGuards({
secrets: dependencies.secrets,
names: [jtiSecretName, nonceSecretName],
value: stringToBytes(DateTime.formatIso(now)),
});
if (!consumedReplayGuards) {
return yield* new EnvironmentHttpConflictError({
message: "Cloud health request was already consumed.",
});
}
const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
const descriptor = yield* dependencies.environment.getDescriptor;
const responseExpiresAt = DateTime.add(now, { minutes: 5 });
const responsePayload = {
iss: `t3-env:${environmentId}`,
aud: normalizeRelayIssuer(relayIssuer),
sub: environmentId,
jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
iat: nowSeconds,
exp: Math.floor(responseExpiresAt.epochMilliseconds / 1_000),
environmentId,
requestNonce: proof.nonce,
status: "online",
descriptor,
checkedAt: DateTime.formatIso(now),
} satisfies RelayEnvironmentHealthResponseProofPayload;
const responseProof = yield* signRelayJwt({
privateKey: keyPair.privateKey,
typ: RELAY_HEALTH_RESPONSE_TYP,
payload: responsePayload,
}).pipe(
Effect.mapError(
(cause) =>
new EnvironmentAuth.ServerAuthCloudHealthJwtSigningError({
cause,
}),
),
);
const response = {
environmentId,
status: "online",
descriptor,
checkedAt: responsePayload.checkedAt,
proof: responseProof,
} satisfies RelayEnvironmentHealthResponseShape;
yield* appendCloudCredentialResponseHeaders;
return response;
},
Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
failEnvironmentCloudInternalError(error.message)(error),
),
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not answer cloud health request."),
),
Effect.catchTag(
"PlatformError",
failEnvironmentCloudInternalError("Could not answer cloud health request."),
),
);
const cloudMintCredentialHandler = Effect.fn("environment.cloud.mintCredential")(
function* (dependencies: CloudHttpDependencies, request: RelayCloudMintCredentialRequest) {
const cloudMintPublicKey = yield* dependencies.secrets
.get(CLOUD_MINT_PUBLIC_KEY)
.pipe(
Effect.flatMap((bytes) =>
Option.isSome(bytes)
? Effect.succeed(bytesToString(bytes.value))
: Effect.fail(new EnvironmentAuth.ServerAuthCloudMintPublicKeyMissingError({})),
),
);
const relayIssuer = yield* dependencies.secrets
.get(RELAY_ISSUER_SECRET)
.pipe(
Effect.flatMap((bytes) =>
Option.isSome(bytes)
? Effect.succeed(bytesToString(bytes.value))
: dependencies.secrets
.get(RELAY_URL_SECRET)
.pipe(
Effect.flatMap((fallbackBytes) =>
Option.isSome(fallbackBytes)
? Effect.succeed(bytesToString(fallbackBytes.value))
: Effect.fail(new EnvironmentAuth.ServerAuthCloudRelayIssuerMissingError({})),
),
),
),
);
const environmentId = yield* dependencies.environment.getEnvironmentId;
const linkedCloudUserId = yield* readInstalledCloudUserId(dependencies.secrets);
const now = yield* DateTime.now;
const nowSeconds = Math.floor(now.epochMilliseconds / 1_000);
const proofOption = yield* verifyRelayJwt({
publicKey: cloudMintPublicKey,
token: request.proof,
typ: RELAY_MINT_REQUEST_TYP,
issuer: normalizeRelayIssuer(relayIssuer),
audience: `t3-env:${environmentId}`,
nowEpochSeconds: nowSeconds,
}).pipe(Effect.flatMap(decodeCloudMintProof), Effect.option);
if (
Option.isNone(proofOption) ||
proofOption.value.environmentId !== environmentId ||
proofOption.value.sub !== linkedCloudUserId ||
proofOption.value.cnf.jkt !== proofOption.value.clientProofKeyThumbprint ||
!hasBoundedCloudProofLifetime({ ...proofOption.value, nowSeconds }) ||
!hasExactScope({ scopes: proofOption.value.scope, expected: "environment:connect" })
) {
return yield* new EnvironmentHttpUnauthorizedError({
message: "Invalid cloud mint request.",
});
}
const proof = proofOption.value;
const jtiSecretName = `${CLOUD_MINT_JTI_PREFIX}${proof.jti}`;
const nonceSecretName = `${CLOUD_MINT_NONCE_PREFIX}${proof.nonce}`;
const consumedReplayGuards = yield* consumeCloudReplayGuards({
secrets: dependencies.secrets,
names: [jtiSecretName, nonceSecretName],
value: stringToBytes(DateTime.formatIso(now)),
});
if (!consumedReplayGuards) {
return yield* new EnvironmentHttpConflictError({
message: "Cloud mint request was already consumed.",
});
}
const keyPair = yield* getOrCreateEnvironmentKeyPairFromSecretStore(dependencies.secrets);
const issued = yield* dependencies.environmentAuth.createPairingLink({
scopes: AuthStandardClientScopes,
subject: "cloud-connect",
ttl: Duration.minutes(2),
label: "T3 Connect connect",
proofKeyThumbprint: proof.clientProofKeyThumbprint,
});
const responsePayload = {
iss: `t3-env:${environmentId}`,
aud: normalizeRelayIssuer(relayIssuer),
sub: environmentId,
jti: yield* Crypto.Crypto.pipe(Effect.flatMap((crypto) => crypto.randomUUIDv4)),
iat: nowSeconds,
exp: Math.floor(issued.expiresAt.epochMilliseconds / 1_000),
environmentId,
clientProofKeyThumbprint: proof.clientProofKeyThumbprint,
requestNonce: proof.nonce,
credential: issued.credential,
} satisfies RelayEnvironmentMintResponseProofPayload;
const responseProof = yield* signRelayJwt({
privateKey: keyPair.privateKey,
typ: RELAY_MINT_RESPONSE_TYP,
payload: responsePayload,
}).pipe(
Effect.mapError(
(cause) =>
new EnvironmentAuth.ServerAuthCloudMintJwtSigningError({
cause,
}),
),
);
const response = {
credential: issued.credential,
expiresAt: DateTime.formatIso(issued.expiresAt),
proof: responseProof,
} satisfies RelayEnvironmentMintResponseShape;
yield* appendCloudCredentialResponseHeaders;
return response;
},
Effect.catchIf(EnvironmentAuth.isServerAuthInternalError, (error) =>
failEnvironmentCloudInternalError(error.message)(error),
),
Effect.catchIf(
ServerSecretStore.isSecretStoreError,
failEnvironmentCloudInternalError("Could not issue cloud connection credential."),
),
Effect.catchTag(
"PlatformError",
failEnvironmentCloudInternalError("Could not issue cloud connection credential."),
),
);
export const connectHttpApiLayer = HttpApiBuilder.group(
EnvironmentHttpApi,
"connect",
Effect.fnUntraced(function* (handlers) {
const dependencies = yield* cloudHttpDependencies;
return handlers
.handle("linkProof", ({ payload }) => cloudLinkProofHandler(dependencies, payload))
.handle("relayConfig", ({ payload }) => cloudRelayConfigHandler(dependencies, payload))
.handle("linkState", () => cloudLinkStateHandler(dependencies))
.handle("unlink", () => cloudUnlinkHandler(dependencies))
.handle("preferences", ({ payload }) => cloudPreferencesHandler(dependencies, payload))
.handle("health", ({ payload }) => cloudEnvironmentHealthHandler(dependencies, payload))
.handle("mintCredential", ({ payload }) => cloudMintCredentialHandler(dependencies, payload))
.handle("t3MintCredential", ({ payload }) =>
traceRelayRequest(cloudMintCredentialHandler(dependencies, payload)),
);
}),
);