scripts/qa-gui-reliability.py

#!/usr/bin/env python3
"""Real GTK reliability probes using an isolated server and a Unix relay.

Requires a display and Python GObject/AT-SPI. Run cargo build --workspace first.
Does not connect to the user's signaltty server. Prints JSON evidence.
"""
import argparse
import base64
import json
import os
from pathlib import Path
import select
import signal
import socket
import subprocess
import tempfile
import threading
import time
from qa_evidence import preserve_logs

import gi

gi.require_version("Atspi", "2.0")
from gi.repository import Atspi, Gio, GLib


def until(probe, seconds=10):
    deadline = time.monotonic() + seconds
    while time.monotonic() < deadline:
        while GLib.MainContext.default().pending():
            GLib.MainContext.default().iteration(False)
        result = probe()
        if result:
            return result
        time.sleep(0.05)
    raise AssertionError("probe timed out")


class Relay:
    def __init__(self, proxy, server):
        self.server = str(server)
        self.offline = threading.Event()
        self.stopped = threading.Event()
        self.listener = socket.socket(socket.AF_UNIX)
        self.listener.bind(str(proxy))
        self.listener.listen()
        self.listener.settimeout(0.1)
        self.thread = threading.Thread(target=self.accept, daemon=True)
        self.thread.start()

    def accept(self):
        while not self.stopped.is_set():
            try:
                client, _ = self.listener.accept()
            except socket.timeout:
                continue
            except OSError:
                break
            threading.Thread(target=self.forward, args=(client,), daemon=True).start()

    def forward(self, client):
        with client, socket.socket(socket.AF_UNIX) as backend:
            try:
                if self.offline.is_set():
                    return
                backend.connect(self.server)
                while not self.offline.is_set() and not self.stopped.is_set():
                    readable, _, _ = select.select([client, backend], [], [], 0.05)
                    for source in readable:
                        data = source.recv(65536)
                        if not data:
                            return
                        (backend if source is client else client).sendall(data)
            except OSError:
                pass

    def close(self):
        self.stopped.set()
        self.listener.close()
        self.thread.join(timeout=1)


def walk(widget):
    widget.clear_cache()
    yield widget
    for child in widget:
        yield from walk(child)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--hold-seconds", type=float, default=0,
                        help="keep the final test window open for visual inspection")
    parser.add_argument("--screenshot", type=Path, help="capture the owned test window using KDE KWin/Spectacle")
    args = parser.parse_args()
    binaries = Path(__file__).resolve().parents[1] / "target/debug"
    bus = Gio.bus_get_sync(Gio.BusType.SESSION)
    owned = bus.call_sync('org.freedesktop.DBus', '/org/freedesktop/DBus',
                         'org.freedesktop.DBus', 'NameHasOwner',
                         GLib.Variant('(s)', ('dev.signaltty.gui',)),
                         GLib.VariantType.new('(b)'), Gio.DBusCallFlags.NONE, 3000, None)
    if owned.unpack()[0]:
        raise RuntimeError('A signaltty GUI already owns this desktop session; close it before native QA.')
    results = []
    with tempfile.TemporaryDirectory(prefix="signaltty-gui-qa-") as tmp:
        root = Path(tmp)
        sock, proxy = root / "server.sock", root / "proxy.sock"
        log = (root / "server.log").open("w")
        gui_log = (root / "gui.log").open("w")
        server = gui = relay = None

        def start_server():
            return subprocess.Popen([str(binaries / "signaltty-server"), "--socket", str(sock),
                "--state-dir", str(root / "state"), "--plugin-dir", str(root / "plugins"),
                "--agents-dir", str(root / "agents")], stdout=log, stderr=log)

        def call(method, **params):
            with socket.socket(socket.AF_UNIX) as connection:
                connection.settimeout(3)
                connection.connect(str(sock))
                connection.sendall((json.dumps(dict(protocol="signaltty/1", id="qa", method=method, params=params)) + "\n").encode())
                response = json.loads(connection.makefile().readline())
                assert response["ok"], response
                return response["result"]

        def ready():
            try:
                return call("server.status")
            except (OSError, ValueError):
                return False

        def app():
            for item in Atspi.get_desktop(0):
                if item.get_process_id() == gui.pid:
                    return item
            return None

        def texts():
            target = app()
            if not target:
                return []
            return [Atspi.Text.get_text(w, 0, -1) for w in walk(target)
                    if "Text" in w.get_interfaces()]

        def terminal_text():
            target = app()
            if not target:
                return ""
            return "\n".join(Atspi.Text.get_text(w, 0, -1) for w in walk(target)
                if w.get_role_name() == "terminal")

        def gtk(method, parameters=None, timeout=1000):
            return bus.call_sync("dev.signaltty.gui", "/dev/signaltty/gui/window/1", "org.gtk.Actions",
                                 method, parameters, None, Gio.DBusCallFlags.NONE, timeout, None)

        def action(name):
            gtk("Activate", GLib.Variant("(sava{sv})", (name, [], {})))

        def input_to(pane, text):
            call("pane.input", pane_id=pane, data_b64=base64.b64encode(text.encode()).decode())

        def pane_get(pane):
            return call("pane.get", pane_id=pane)["pane"]

        def record(case, **evidence):
            result = dict(case=case, status="PASS", evidence=evidence)
            results.append(result)
            print(json.dumps(result), flush=True)

        def capture(path):
            assert gui.poll() is None, "test GUI already closed"
            focus_script = root / "focus.js"
            focus_script.write_text("const windows = workspace.windowList().filter(w => w.resourceClass === 'dev.signaltty.gui' || w.resourceClass === 'signaltty-gui'); if (windows.length) workspace.activeWindow = windows[windows.length - 1];")
            name = f"signaltty-qa-{gui.pid}"
            script_id = subprocess.check_output(["qdbus6", "org.kde.KWin", "/Scripting", "org.kde.kwin.Scripting.loadScript", str(focus_script), name], text=True).strip()
            try:
                subprocess.run(["qdbus6", "org.kde.KWin", f"/Scripting/Script{script_id}", "org.kde.kwin.Script.run"], check=True, stdout=subprocess.DEVNULL)
                assert gui.poll() is None
                subprocess.run(["spectacle", "--background", "--activewindow", "--nonotify", "--output", str(path.resolve())], check=True)
            finally:
                subprocess.run(["qdbus6", "org.kde.KWin", "/Scripting", "org.kde.kwin.Scripting.unloadScript", name], check=True, stdout=subprocess.DEVNULL)

        try:
            server = start_server()
            until(ready)
            ws = call("workspace.create", name="Reliability", cwd=tmp)["workspace"]["id"]
            pane = call("pane.spawn", workspace_id=ws, argv=["sh"], agent_hint="codex")["pane"]["id"]
            input_to(pane, "echo QASTART\n")
            relay = Relay(proxy, sock)
            gui = subprocess.Popen([str(binaries / "signaltty-gui"), "--socket", str(proxy)],
                stdout=gui_log, stderr=gui_log, env=dict(os.environ, SIGNALTTY_NOTIFY="0", SIGNALTTY_REFRESH_METRICS="1"))
            until(lambda: app() and "QASTART" in terminal_text())
            before = len(app())
            subprocess.run([str(binaries / "signaltty-gui"), "--socket", str(proxy)],
                           stdout=gui_log, stderr=gui_log, timeout=5, check=True)
            time.sleep(0.2)
            after = len(app())
            assert before == after == 1, (before, after)
            record("single_window_activation", before=before, after=after)

            other_ws = call("workspace.create", name="Approval", cwd=tmp)["workspace"]["id"]
            other = call("pane.spawn", workspace_id=other_ws, argv=["cat"], agent_hint="codex")["pane"]["id"]
            call("hook-event", agent="codex", event="PermissionRequest", pane_id=other,
                 decision=dict(id="qa-approval", prompt="QA approval needed", options=[dict(id="allow", label="Allow"), dict(id="deny", label="Deny")]))
            until(lambda: "Approval" in texts())
            action("next-attention")
            until(lambda: "QA approval needed" in texts())
            current = pane_get(other)
            assert current["pending_decision"]["id"] == "qa-approval" and current["attention"] == "permission_required", current
            allow = until(lambda: next((w for w in walk(app()) if "Action" in w.get_interfaces() and w.get_name() == "Allow"), None))
            Atspi.Action.do_action(allow, 0)
            until(lambda: pane_get(other).get("pending_decision") is None and pane_get(other)["attention"] == "none")
            record("approval_survives_focus_and_can_be_answered", decision_before_answer=current["pending_decision"], attention=current["attention"])

            # Keep the active approval terminal for reconnect, so the same VTE
            # must recover outage data. cat provides deterministic byte echo.
            input_to(other, "QARECONNECTA\n")
            until(lambda: "QARECONNECTA" in terminal_text())
            relay.offline.set()
            time.sleep(0.3)
            input_to(other, "QARECONNECTB\n")
            until(lambda: "QARECONNECTB" in call("pane.read", pane_id=other)["text"])
            relay.offline.clear()
            until(lambda: "QARECONNECTB" in terminal_text())
            input_to(other, "QAONLINE\n")
            content = until(lambda: terminal_text() if "QAONLINE" in terminal_text() else False)
            assert all(content.count(marker) == 2 for marker in ["QARECONNECTA", "QARECONNECTB", "QAONLINE"]), content
            record("reconnect_restores_terminal", terminal_text=content)

            os.kill(server.pid, signal.SIGSTOP)
            try:
                action("new-tab")
                started = time.monotonic()
                gtk("List")
                elapsed = time.monotonic() - started
                assert elapsed < 1, elapsed
            finally:
                os.kill(server.pid, signal.SIGCONT)
            record("stalled_server_keeps_gtk_responsive", seconds=elapsed)
            # Let the in-flight new tab finish before checking layout actions.
            until(lambda: len(call("workspace.get", workspace_id=other_ws)["panes"]) == 2)
            action("split-right")
            until(lambda: len(call("workspace.get", workspace_id=other_ws)["panes"]) == 3)
            action("close-pane")
            until(lambda: len(call("workspace.get", workspace_id=other_ws)["panes"]) == 2)
            record("split_and_close_after_recovery", owned_panes=2)

            restored_ws = call("workspace.create", name="Restored", cwd=tmp)["workspace"]["id"]
            restored = call("pane.spawn", workspace_id=restored_ws, argv=["sleep", "600"], agent_hint="codex")["pane"]["id"]
            call("hook-event", agent="codex", event="PreToolUse", pane_id=restored)
            until(lambda: "Working" in texts())
            call("server.shutdown", force=True)
            server.wait(timeout=5)
            server = start_server()
            until(ready)
            until(lambda: "Exited" in texts() and "Working" not in texts() and "Running" not in texts())
            current = pane_get(restored)
            assert current["live"]["state"] == "exited", current
            record("restored_status_is_exited", live=current["live"], saved_lifecycle=current["lifecycle"], ui_text=texts())
            print(json.dumps(dict(inspect_pid=gui.pid, temp_dir=tmp)), flush=True)
            if args.screenshot:
                capture(args.screenshot)
            if args.hold_seconds:
                time.sleep(args.hold_seconds)
        except Exception:
            if args.screenshot and gui and gui.poll() is None:
                capture(args.screenshot.with_name(args.screenshot.stem + "-failure.png"))
            print(json.dumps({"debug_ui": [(w.get_role_name(), w.get_name()) for w in walk(app())] if app() else [], "gui_log": (root / "gui.log").read_text(), "terminal_text": terminal_text(), "server_terminal": call("pane.read", pane_id=other) if "other" in locals() else None}), flush=True)
            raise
        finally:
            if server and server.poll() is None:
                os.kill(server.pid, signal.SIGCONT)
                try:
                    call("server.shutdown", force=True)
                    server.wait(timeout=5)
                except (OSError, ValueError, subprocess.TimeoutExpired):
                    server.kill()
                    server.wait()
            if gui and gui.poll() is None:
                gui.terminate()
                gui.wait(timeout=5)
            if relay:
                relay.close()
            log.close()
            gui_log.close()
            preserve_logs(root, 'accessibility')


if __name__ == "__main__":
    main()