server/sync.test.ts

import { test, mock } from "node:test";
import assert from "node:assert/strict";
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { spawn } from "node:child_process";
import { google } from "./google-api.js";
import { ImapFlow } from "imapflow";
import nodemailer from "nodemailer";
import { quietImapWatch, quietGmailRevision } from "./imap-watch-fixture.js";
import type { Mail, State } from "../shared/types.js";
import type { StoredAccount } from "./store.js";

function deferred() {
  let resolve!: () => void;
  const promise = new Promise<void>((r) => { resolve = r; });
  return { promise, resolve };
}
const fixture = (id: string, provider: StoredAccount["provider"] = "imap"): StoredAccount => ({
  id, email: `${id}@example.com`, provider, credentials: {},
});
const mail = (a: StoredAccount, id = "one"): Mail => ({
  id: `${a.id}:${id}`, accountId: a.id, remoteId: id,
  sender: "Synthetic Sender", address: "sender@example.com", to: a.email,
  subject: "Please review", preview: "Synthetic mail", body: "Synthetic body",
  date: 1700000000000, unread: true, category: "important", relevance: 2,
  reason: "Saved sorting", archived: false, snoozedUntil: null,
});
const aiAnswer = () => new Response(JSON.stringify({ model: "synthetic", answers: {
  category: { type: "choice", choice: "other" }, relevance: { type: "score", score: 0 }, reason: { type: "choice", choice: "Newsletter" },
} }));

// The subprocess releases SQLite handles before the parent removes its profile.
if (!process.env.EMC2_SYNC_TEST_CHILD) {
  test("synthetic sync service regressions", async () => {
    const directory = mkdtempSync(path.join(tmpdir(), "emc2-sync-"));
    try {
      const env: NodeJS.ProcessEnv = { ...process.env, EMC2_SYNC_TEST_CHILD: "1", MAIL_DATA_DIR: directory,
        MAIL_CONFIG_PATH: path.join(directory, "absent.env"), MAIL_DESKTOP_TOKEN: "synthetic-token",
        GOOGLE_CLIENT_ID: "synthetic-client", GOOGLE_CLIENT_SECRET: "synthetic-secret",
        TYPESAFE_API_KEY: "", OPENAI_API_KEY: "" };
      delete env.NODE_TEST_CONTEXT;
      // Async spawn keeps the parent's event loop free, and the bound sits well inside --test-timeout=120000 so a
      // cold tsx start on a loaded CI runner is not killed mid-run (the old 30s spawnSync bound was the flake).
      const child = spawn(process.execPath, ["--import", "tsx", "--test", fileURLToPath(import.meta.url)], { env, stdio: ["ignore", "pipe", "pipe"] });
      let stdout = "", stderr = "";
      child.stdout.setEncoding("utf8").on("data", (chunk: string) => { stdout += chunk; });
      child.stderr.setEncoding("utf8").on("data", (chunk: string) => { stderr += chunk; });
      const timer = setTimeout(() => child.kill("SIGKILL"), 100000);
      const [status, signal] = await new Promise<[number | null, NodeJS.Signals | null]>((resolve, reject) => {
        child.once("error", reject).once("close", (code, sig) => resolve([code, sig]));
      }).finally(() => clearTimeout(timer));
      const result = { stdout, stderr, status, signal };
      assert.equal(result.status, 0, `child exited with status ${result.status} signal ${result.signal}\n${result.stdout}\n${result.stderr}`);
      assert.match(result.stdout, /account connection, progress, coalescing and recovery/);
      console.log(result.stdout.trim());
    } finally {
      rmSync(directory, { recursive: true, force: true });
    }
  });
} else {
  test("account connection, progress, coalescing and recovery", async (t) => {
    const { providers } = await import("./providers.js");
    const { all, put, remove } = await import("./store.js");
    const { startServer } = await import("./index.js");
    mock.method(ImapFlow.prototype, "connect", async () => {});
    mock.method(ImapFlow.prototype, "mailboxOpen", async () => ({}));
    mock.method(ImapFlow.prototype, "close", () => {});
    mock.method(nodemailer, "createTransport", () => ({ verify: async () => true }));
    mock.method(google, "gmail", () => ({ users: { labels: { list: async () => ({ data: { labels: [] } }) } } }), {});
    let onMailChanged: (() => void) | undefined;
    const server = await startServer(0, undefined, () => onMailChanged?.(), undefined, quietImapWatch, quietGmailRevision);
    const address = server.address();
    assert.ok(address && typeof address === "object");
    const origin = `http://127.0.0.1:${address.port}`;
    const headers = { "X-Desktop-Token": "synthetic-token", "Content-Type": "application/json", "X-CSRF-Token": "" };
    const networkFetch = globalThis.fetch;
    const state = async () => (await (await networkFetch(`${origin}/api/state`, { headers })).json()) as State & { csrf: string };
    headers["X-CSRF-Token"] = (await state()).csrf;
    const request = (route: string, body = {}, method = "POST") => networkFetch(`${origin}/api${route}`, { method, headers, body: JSON.stringify(body) });
    const add = (email: string) => request("/accounts/imap", {
      provider: "imap", email, password: "synthetic-password", imapHost: "imap.example.com", smtpHost: "smtp.example.com",
    });
    const clear = () => {
      for (const kind of ["account", "mail", "rule"] as const)
        for (const record of all(kind)) remove(kind, record.id);
    };
    t.beforeEach(clear);
    try {
      await t.test("second and third accounts start while the first is pending; duplicate sync coalesces and guards remain", async () => {
        const first = fixture("first");
        put("account", first.id, first);
        const entered = deferred(), release = deferred(), secondStarted = deferred(), releaseSecond = deferred(), thirdStarted = deferred();
        const calls: string[] = [];
        let active = 0, peak = 0;
        providers.imap.list = async (a) => {
          calls.push(a.email); peak = Math.max(peak, ++active);
          if (a.id === first.id) { entered.resolve(); await release.promise; }
          else if (a.email === "second@example.com") { secondStarted.resolve(); await releaseSecond.promise; }
          else thirdStarted.resolve();
          active--;
          return [];
        };
        const pending = request("/sync");
        await entered.promise;
        let duplicate: Promise<Response> | undefined;
        let queuedRead: Promise<Response> | undefined;
        try {
          assert.equal((await add("second@example.com")).status, 200);
          await secondStarted.promise;
          assert.equal((await add("third@example.com")).status, 200);
          assert.equal(calls.length, 2, "the third account must wait for a slot");
          releaseSecond.resolve();
          await thirdStarted.promise;
          assert.equal((await state()).syncing, true);
          duplicate = request("/sync");
          assert.equal((await request("/rules", { sender: "sender@example.com", instruction: "Keep", category: "important", relevance: 2 })).status, 409);
          queuedRead = request(`/messages/${first.id}:one`, { action: "read" }, "PATCH");
          assert.equal((await networkFetch(`${origin}/api/sync`, { method: "POST", headers: { "X-Desktop-Token": "synthetic-token" } })).status, 403);
          assert.equal((await networkFetch(`${origin}/api/state`)).status, 403);
          assert.equal((await networkFetch(`${origin}/api/state`, { headers: { ...headers, Origin: "https://untrusted.example" } })).status, 403);
        } finally {
          release.resolve();
          releaseSecond.resolve();
          if (queuedRead) assert.equal((await queuedRead).status, 404);
          await pending;
          if (duplicate) assert.equal((await duplicate).status, 200);
        }
        assert.equal(calls.length, 3);
        assert.equal(peak, 2);
        assert.equal((await state()).syncing, false);
        clear();
      });
      await t.test("Google callback schedules during pending sync and reconnect cannot restore old credentials", async () => {
        const a = { ...fixture("google", "gmail"), credentials: { refreshToken: "old-synthetic" } };
        put("account", a.id, a);
        const entered = deferred(), release = deferred(), newAccountStarted = deferred();
        const tokens: string[] = [];
        providers.gmail.list = async (account) => {
          tokens.push(account.credentials.refreshToken!);
          if (tokens.length === 1) { entered.resolve(); await release.promise; }
          if (account.id !== a.id) newAccountStarted.resolve();
          return [];
        };
        mock.method(google.auth.OAuth2.prototype, "getToken", async () => ({ tokens: { refresh_token: "new-synthetic" } }));
        let profileEmail = a.email;
        const gmailMock = mock.method(google, "gmail", () => ({ users: { getProfile: async () => ({ data: { emailAddress: profileEmail } }), labels: { list: async () => ({ data: { labels: [] } }) } } }), {});
        const connectGoogle = async () => {
          const { url: launch } = await (await request("/oauth/google/start")).json() as { url: string };
          const entry = await networkFetch(`${origin}${launch}`, { redirect: "manual" });
          const location = entry.headers.get("location")!;
          const oauthState = new URL(location).searchParams.get("state")!;
          const cookie = entry.headers.get("set-cookie")!.split(";")[0];
          const callback = await networkFetch(`${origin}/api/oauth/google/callback?state=${oauthState}&code=synthetic-code`, { headers: { Cookie: cookie } });
          assert.equal(callback.status, 200);
        };
        const pending = request("/sync");
        await entered.promise;
        try {
          await connectGoogle();
          assert.equal(all("account")[0].credentials.refreshToken, "new-synthetic");
          profileEmail = "oauth-second@example.com";
          await connectGoogle();
          await newAccountStarted.promise;
          assert.equal(all("account").length, 2);
        } finally { release.resolve(); await pending; gmailMock.mock.restore(); }
        assert.deepEqual(tokens, ["old-synthetic", "new-synthetic", "new-synthetic"]);
        assert.equal(all("account").find((account) => account.id === a.id)?.credentials.refreshToken, "new-synthetic");
        clear();
      });
      await t.test("fetched Gmail batches are visible while later fetching and every configured AI judgement are pending", async () => {
        const a = fixture("progress", "gmail"); put("account", a.id, a);
        const fetched = deferred(), finishFetch = deferred(), sorting = deferred(), finishSorting = deferred();
        providers.gmail.list = async (account, _cached, onBatch) => {
          const batch = [mail(account)]; onBatch?.(batch); fetched.resolve();
          await finishFetch.promise;
          return [...batch, ...Array.from({ length: 17 }, (_, i) => mail(account, String(i)))];
        };
        let modelCalls = 0, modelActive = 0, modelPeak = 0;
        process.env.TYPESAFE_API_KEY = "synthetic-key";
        globalThis.fetch = async (url, init) => {
          if (String(url).startsWith(origin)) return networkFetch(url, init);
          assert.equal(String(url), "https://api.typesafe.ai/v1/systemone");
          modelCalls++; modelPeak = Math.max(modelPeak, ++modelActive); sorting.resolve(); await finishSorting.promise;
          modelActive--; return aiAnswer();
        };
        const pending = request("/sync");
        await fetched.promise;
        try {
          assert.equal((await state()).messages.length, 1);
          assert.equal((await state()).messages[0].reason, "May need your attention");
          finishFetch.resolve(); await sorting.promise;
          assert.equal((await state()).messages.length, 18);
          assert.equal((await state()).syncing, true);
        } finally {
          finishFetch.resolve(); finishSorting.resolve(); await pending;
          globalThis.fetch = networkFetch; process.env.TYPESAFE_API_KEY = "";
        }
        assert.equal(modelCalls, 18); assert.equal(modelPeak, 6);
        console.log("synthetic Gmail first sync: 1 visible during fetch, 18 visible during AI, 18 AI calls, peak 6");
        assert.ok((await state()).messages.every((m) => m.reason === "Newsletter"));
        clear();
      });
      await t.test("provider failure keeps cached mail, exposes error, clears flags and allows retry", async () => {
        const a = fixture("failure"); put("account", a.id, a);
        const cached = mail(a); put("mail", cached.id, cached);
        providers.imap.list = async () => { throw new Error("Synthetic provider timeout"); };
        const response = await request("/sync");
        assert.equal(response.status, 200);
        const result = await response.json() as { errors: string[] };
        assert.equal(result.errors.length, 1);
        assert.deepEqual(all("mail"), [cached]);
        assert.equal((await state()).syncing, false);
        assert.ok((await state()).accounts[0].error);
        providers.imap.list = async () => [cached];
        assert.equal((await request("/sync")).status, 200);
        assert.equal((await state()).accounts[0].error, undefined);
        clear();
      });
      await t.test("background connection reports AI failure in account state and retains local mail", async () => {
        let syncCalls = 0;
        providers.imap.list = async (a) => { syncCalls++; return [mail(a)]; };
        const failed = deferred(), release = deferred(), completed = deferred();
        process.env.TYPESAFE_API_KEY = "synthetic-key";
        globalThis.fetch = async () => { failed.resolve(); await release.promise; return new Response("", { status: 503 }); };
        let pending: Promise<void> | undefined;
        try {
          assert.equal((await add("ai-failure@example.com")).status, 200);
          await failed.promise;
          // A second sync can arrive after completion and clear the warning being tested.
          onMailChanged = completed.resolve;
          pending = completed.promise;
          assert.equal((await state()).syncing, true);
          release.resolve();
          await pending;
          const result = await state();
          assert.equal(result.syncing, false);
          assert.equal(syncCalls, 1);
          assert.match(result.accounts[0].error || "", /Jev could not sort mail/);
          assert.equal(result.messages[0].reason, "May need your attention");
        } finally {
          release.resolve(); if (pending) await pending;
          onMailChanged = undefined;
          globalThis.fetch = networkFetch; process.env.TYPESAFE_API_KEY = "";
        }
        clear();
      });
      await t.test("partial fetch failure preserves cached mail and still sorts fetched mail; successful retry reconciles archive and snooze", async () => {
        const a = fixture("partial"); put("account", a.id, a);
        const cached = { ...mail(a, "cached"), snoozedUntil: 1800000000000 };
        const absent = mail(a, "absent");
        put("mail", cached.id, cached); put("mail", absent.id, absent);
        providers.imap.list = async (account, _cached, onBatch) => {
          onBatch?.([mail(account)]); throw new Error("Synthetic later-page failure");
        };
        let modelCalls = 0;
        process.env.TYPESAFE_API_KEY = "synthetic-key";
        globalThis.fetch = async () => { modelCalls++; return aiAnswer(); };
        try { assert.equal((await request("/sync")).status, 200); }
        finally { globalThis.fetch = networkFetch; process.env.TYPESAFE_API_KEY = ""; }
        assert.equal(modelCalls, 1);
        assert.deepEqual(all("mail").find((m) => m.id === cached.id), cached);
        assert.equal(all("mail").find((m) => m.id === absent.id)?.archived, false);
        assert.ok(all("mail").some((m) => m.id === `${a.id}:one`));
        assert.equal(all("mail").find((m) => m.id === `${a.id}:one`)?.reason, "Newsletter");
        providers.imap.list = async () => [{ ...cached, unread: false }, mail(a)];
        await request("/sync");
        const updated = all("mail").find((m) => m.id === cached.id)!;
        assert.equal(updated.unread, false); assert.equal(updated.snoozedUntil, cached.snoozedUntil);
        assert.equal(updated.reason, cached.reason);
        assert.equal(all("mail").find((m) => m.id === absent.id)?.archived, true);
        clear();
      });
      await t.test("late completion cannot recreate a removed account or its mail", async () => {
        const a = fixture("removed"); put("account", a.id, a);
        const entered = deferred(), release = deferred();
        providers.imap.list = async () => { entered.resolve(); await release.promise; return [mail(a)]; };
        const pending = request("/sync"); await entered.promise;
        remove("account", a.id); release.resolve(); await pending;
        assert.deepEqual(all("account"), []); assert.deepEqual(all("mail"), []);
      });
    } finally {
      globalThis.fetch = networkFetch;
      mock.restoreAll();
      await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
    }
  });
}