apps/mobile/scripts/measure-renders.mts
// See docs/internals/mobile-development.md#measuring-renders.
import * as NodeHttpClient from "@effect/platform-node/NodeHttpClient";
import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
import * as NodeServices from "@effect/platform-node/NodeServices";
import * as NodeSocket from "@effect/platform-node/NodeSocket";
import {
CommandId,
EnvironmentId,
ExecutionEnvironmentDescriptor,
ORCHESTRATION_PROTOCOL_HEADER,
ORCHESTRATION_PROTOCOL_VERSION,
OrchestrationV2ShellSnapshot,
ProjectId,
ProviderInstanceId,
ThreadId,
WsRpcGroup,
type OrchestrationV2ThreadProjection,
} from "@t3tools/contracts";
import * as Console from "effect/Console";
import * as Crypto from "effect/Crypto";
import * as Effect from "effect/Effect";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Option from "effect/Option";
import * as Schema from "effect/Schema";
import * as Stream from "effect/Stream";
import { Command, Flag } from "effect/cli";
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
import { RpcClient, RpcSerialization } from "effect/rpc";
import { Socket } from "effect/socket";
const RenderReport = Schema.Struct({
commits: Schema.Number,
totalRenders: Schema.Number,
components: Schema.Array(Schema.Struct({ key: Schema.String, count: Schema.Number })),
causes: Schema.Array(Schema.Struct({ key: Schema.String, count: Schema.Number })),
});
type RenderReport = typeof RenderReport.Type;
const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(RenderReport));
const InspectorTargets = Schema.Array(
Schema.Struct({ appId: Schema.String, webSocketDebuggerUrl: Schema.String }),
);
const CdpReply = Schema.Struct({
id: Schema.Number,
result: Schema.optional(
Schema.Struct({
result: Schema.optional(Schema.Struct({ value: Schema.optional(Schema.Unknown) })),
exceptionDetails: Schema.optional(Schema.Unknown),
}),
),
});
const decodeCdpReply = Schema.decodeUnknownOption(Schema.fromJsonString(CdpReply));
const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown));
class MeasureError extends Schema.TaggedError<MeasureError>()("MeasureError", {
message: Schema.String,
}) {}
const STREAM_PROMPT =
"Think it through step by step, then write a 3000-word essay on the history of typesetting in plain paragraphs with a few headings. Do not use any tools.";
interface Options {
readonly server: string;
readonly token: string;
readonly metro: string;
readonly scheme: string;
readonly runs: number;
readonly windowSeconds: number;
readonly provider: string;
readonly model: string;
readonly projectRoot: string;
}
const uuid = Effect.flatMap(Crypto.Crypto, (crypto) => crypto.randomUUIDv4());
const median = (values: ReadonlyArray<number>) => {
const sorted = [...values].sort((a, b) => a - b);
const middle = Math.floor(sorted.length / 2);
return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
};
const connectInspector = Effect.fn("measure.connectInspector")(function* (metro: string) {
const http = yield* HttpClient.HttpClient;
const targets = yield* http
.get(`${metro}/json/list`)
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(InspectorTargets)));
const target = targets.find((entry) => entry.appId.startsWith("com.t3tools.t3code"));
if (!target)
return yield* new MeasureError({ message: `No T3 Code app is attached to Metro at ${metro}.` });
// Expo closes inspector sockets unless Origin matches Metro's advertised URL; pass that URL as --metro.
const socket = yield* Socket.makeWebSocket(target.webSocketDebuggerUrl, {
headers: { origin: new URL(metro).origin },
openTimeout: "10 seconds",
});
const writer = yield* socket.writer;
const reader = yield* socket.reader;
let nextId = 0;
return Effect.fn("measure.evaluate")(function* (expression: string) {
const id = ++nextId;
yield* writer.write(
encodeJson({ id, method: "Runtime.evaluate", params: { expression, returnByValue: true } }),
);
while (true) {
for (const frame of yield* reader.pull) {
const text = typeof frame === "string" ? frame : new TextDecoder().decode(frame);
const reply = decodeCdpReply(text);
if (Option.isNone(reply) || reply.value.id !== id) continue;
if (reply.value.result?.exceptionDetails)
return yield* new MeasureError({
message: `Evaluation failed: ${encodeJson(reply.value.result.exceptionDetails)}`,
});
return reply.value.result?.result?.value;
}
}
});
});
const connectServer = Effect.fn("measure.connectServer")(function* (options: Options) {
const http = (yield* HttpClient.HttpClient).pipe(
HttpClient.mapRequest((request) =>
request.pipe(
HttpClientRequest.bearerToken(options.token),
HttpClientRequest.setHeader(
ORCHESTRATION_PROTOCOL_HEADER,
String(ORCHESTRATION_PROTOCOL_VERSION),
),
),
),
HttpClient.filterStatusOk,
);
const descriptor = yield* http
.get(`${options.server}/.well-known/t3/environment`)
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)));
const shell = yield* http
.get(`${options.server}/api/orchestration/shell`)
.pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(OrchestrationV2ShellSnapshot)));
let projectId = shell.projects.find(
(project) => project.workspaceRoot === options.projectRoot,
)?.id;
// A project record outlives its folder, and turns fail without it.
yield* (yield* FileSystem.FileSystem).makeDirectory(options.projectRoot, { recursive: true });
if (!projectId) {
projectId = ProjectId.make(yield* uuid);
yield* http.execute(
HttpClientRequest.post(`${options.server}/api/projects/mutate`).pipe(
HttpClientRequest.bodyJsonUnsafe({
type: "project.create",
commandId: yield* uuid,
projectId,
title: "Render measurement",
workspaceRoot: options.projectRoot,
}),
),
);
}
const ticket = yield* http
.execute(HttpClientRequest.post(`${options.server}/api/auth/websocket-ticket`))
.pipe(
Effect.flatMap(HttpClientResponse.schemaBodyJson(Schema.Struct({ ticket: Schema.String }))),
);
// Built into the caller's scope: `Effect.provide` would close the socket once `make` returns.
const protocol = yield* Layer.build(
RpcClient.layerProtocolSocket().pipe(
Layer.provide(
Socket.layerWebSocket(
`${options.server.replace(/^http/, "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL_VERSION}&wsTicket=${encodeURIComponent(ticket.ticket)}`,
),
),
Layer.provide(RpcSerialization.layerJson),
),
);
const client = yield* RpcClient.make(WsRpcGroup).pipe(Effect.provideContext(protocol));
return { client, environmentId: EnvironmentId.make(descriptor.environmentId), projectId };
});
// Some providers stream only reasoning, so any streaming text item counts.
const streamingItem = (projection: OrchestrationV2ThreadProjection) =>
projection.turnItems.find(
(item) => (item.type === "assistant_message" || item.type === "reasoning") && item.streaming,
);
const measureStreamReply = Effect.fn("measure.streamReply")(function* (
options: Options,
run: number,
route: Option.Option<string>,
) {
if (Option.isSome(route)) yield* reloadAndOpen(options, route.value);
const evaluate = yield* connectInspector(options.metro);
const { client, environmentId, projectId } = yield* connectServer(options);
const threadId = ThreadId.make(yield* uuid);
yield* client["orchestration.launchThread"]({
commandId: CommandId.make(yield* uuid),
threadId,
projectId,
title: `Render measurement ${run}`,
modelSelection: {
instanceId: ProviderInstanceId.make(options.provider),
model: options.model,
},
// The prompt needs no tools; approval-required keeps a stray call from acting.
runtimeMode: "approval-required",
interactionMode: "default",
workspaceStrategy: { type: "root" },
initialMessage: { text: STREAM_PROMPT, attachments: [] },
});
// A failed or interrupted window must not leave a provider turn running.
yield* Effect.addFinalizer(() =>
Effect.gen(function* () {
const after = yield* client["orchestration.getThreadProjection"]({ threadId });
const activeRun = after.runs.find(
(entry) => !["completed", "failed", "interrupted", "cancelled"].includes(entry.status),
);
if (!activeRun) return;
yield* client["orchestration.dispatchCommand"]({
type: "run.interrupt",
commandId: CommandId.make(yield* uuid),
threadId,
runId: activeRun.id,
});
}).pipe(Effect.ignore),
);
if (Option.isNone(route))
yield* evaluate(
`__t3RenderCounterOpen(${encodeJson(`${options.scheme}://threads/${environmentId}/${threadId}`)})`,
);
const projectionOf = (item: { kind: string; projection?: OrchestrationV2ThreadProjection }) =>
item.kind === "snapshot" && item.projection
? Effect.succeed(item.projection)
: client["orchestration.getThreadProjection"]({ threadId });
const streaming = yield* client["orchestration.subscribeThread"]({ threadId }).pipe(
Stream.mapEffect(projectionOf),
Stream.filter((projection) => streamingItem(projection) !== undefined),
Stream.runHead,
Effect.timeout("90 seconds"),
);
if (Option.isNone(streaming))
return yield* new MeasureError({ message: `Run ${run}: the reply never started streaming.` });
yield* evaluate("__t3RenderCounterStart()");
yield* Effect.sleep(`${options.windowSeconds} seconds`);
const report = yield* decodeReport(yield* evaluate("JSON.stringify(__t3RenderCounterStop())"));
// An in-view streaming thread must commit; out-of-view streaming may commit nothing.
if (report.commits === 0 && Option.isNone(route))
return yield* new MeasureError({
message: `Run ${run}: no React commits in the window. Check the device for a red box.`,
});
const after = yield* client["orchestration.getThreadProjection"]({ threadId });
return { report, stillStreaming: streamingItem(after) !== undefined };
}, Effect.scoped);
const appLink = (options: Options, route: string) =>
`__t3RenderCounterOpen(${encodeJson(`${options.scheme}://${route.replace(/^\//, "")}`)})`;
const reloadAndOpen = Effect.fn("measure.reloadAndOpen")(function* (
options: Options,
route: string,
) {
// A link to an already-open screen only refocuses it, so each run starts from a reload.
yield* Effect.scoped(
Effect.flatMap(connectInspector(options.metro), (evaluate) =>
evaluate("__t3RenderCounterReload()"),
),
);
yield* Effect.sleep("15 seconds");
if (route === "/" || route === "") return;
yield* Effect.scoped(
Effect.flatMap(connectInspector(options.metro), (evaluate) =>
evaluate(appLink(options, route)),
),
);
yield* Effect.sleep("5 seconds");
});
const measureRouteOpen = Effect.fn("measure.routeOpen")(function* (
options: Options,
route: string,
run: number,
) {
yield* Effect.scoped(
Effect.flatMap(connectInspector(options.metro), (evaluate) =>
evaluate("__t3RenderCounterReload()"),
),
);
yield* Effect.sleep("15 seconds");
const evaluate = yield* connectInspector(options.metro);
yield* evaluate("__t3RenderCounterStart()");
yield* evaluate(appLink(options, route));
yield* Effect.sleep(`${options.windowSeconds} seconds`);
const report = yield* decodeReport(yield* evaluate("JSON.stringify(__t3RenderCounterStop())"));
if (report.commits === 0)
return yield* new MeasureError({
message: `Run ${run}: no React commits in the window. Check that ${route} is a valid app path.`,
});
return { report, stillStreaming: undefined };
}, Effect.scoped);
const medianTable = (
title: string,
reports: ReadonlyArray<RenderReport>,
pick: (report: RenderReport) => RenderReport["causes"],
limit: number,
) => {
const keys = new Set(reports.flatMap((report) => pick(report).map((entry) => entry.key)));
const rows = [...keys]
.map((key) => ({
key,
count: median(reports.map((report) => pick(report).find((e) => e.key === key)?.count ?? 0)),
}))
.filter((row) => row.count > 0)
.sort((a, b) => b.count - a.count)
.slice(0, limit);
return [
`| ${title} | Renders |`,
"|---|---:|",
...rows.map((r) => `| \`${r.key}\` | ${r.count} |`),
];
};
const formatTable = (reports: ReadonlyArray<RenderReport>) => {
const totals = (pick: (report: RenderReport) => number) => median(reports.map(pick));
return [
`Median of ${reports.length} runs.`,
"",
"| | |",
"|---|---:|",
`| renders | ${totals((r) => r.totalRenders)} |`,
`| commits | ${totals((r) => r.commits)} |`,
// Provider pace sets how many commits land in the window; per-commit cost compares across runs.
`| renders per commit | ${totals((r) => Math.round(r.totalRenders / Math.max(r.commits, 1)))} |`,
"",
...medianTable("Component", reports, (r) => r.components, 20),
"",
...medianTable("Cause", reports, (r) => r.causes, 15),
].join("\n");
};
const command = Command.make(
"measure-renders",
{
server: Flag.String("server").pipe(
Flag.withDescription("T3 server origin the app is paired with."),
Flag.withDefault(""),
),
token: Flag.String("token").pipe(
Flag.withDescription("Bearer token from `t3 auth session issue --token-only`."),
Flag.withDefault(""),
),
route: Flag.String("route").pipe(
Flag.withDescription(
"App path to open and measure. With --server, the path stays open while a new thread streams.",
),
Flag.optional,
),
metro: Flag.String("metro").pipe(Flag.withDefault("http://127.0.0.1:8081")),
scheme: Flag.String("scheme").pipe(Flag.withDefault("t3code-dev")),
runs: Flag.Int("runs").pipe(Flag.withDefault(3)),
window: Flag.Int("window").pipe(
Flag.withDescription("Seconds to record; defaults to 12 for a reply and 6 for --route."),
Flag.optional,
),
provider: Flag.String("provider").pipe(Flag.withDefault("claudeAgent")),
model: Flag.String("model").pipe(Flag.withDefault("claude-haiku-5-5")),
projectRoot: Flag.String("project-root").pipe(
Flag.withDescription("Scratch directory for the measurement project; created if missing."),
Flag.withDefault(""),
),
},
Effect.fn("measure.main")(function* (flags) {
const route = flags.route;
const options: Options = {
...flags,
windowSeconds: Option.getOrElse(flags.window, () =>
Option.isSome(route) && !flags.server ? 6 : 12,
),
};
const streams = Option.isNone(route) || flags.server !== "";
if (streams && (!flags.server || !flags.token || !flags.projectRoot))
return yield* new MeasureError({
message: "Streaming a reply needs --server, --token and --project-root.",
});
const reports: RenderReport[] = [];
for (let run = 1; run <= flags.runs; run++) {
const { report, stillStreaming } =
Option.isSome(route) && !streams
? yield* measureRouteOpen(options, route.value, run)
: yield* measureStreamReply(options, run, route);
yield* Console.error(
`run ${run}: ${report.totalRenders} renders in ${report.commits} commits${stillStreaming === false ? " (warning: streaming ended inside the window)" : ""}`,
);
reports.push(report);
// Let the interrupted turn settle so the next run starts from a quiet app.
yield* Effect.sleep("3 seconds");
}
yield* Console.log(formatTable(reports));
}),
);
Command.run(command, { version: "0.0.0" }).pipe(
Effect.provide(
Layer.mergeAll(
NodeServices.layer,
NodeHttpClient.layerUndici,
NodeSocket.layerWebSocketConstructorWS,
),
),
NodeRuntime.runMain,
);