scripts/verify.py
#!/usr/bin/env python3
"""Verify signaltty; full includes isolated GTK tests, desktop adds native AT-SPI QA."""
import argparse
import json
import os
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import time
from verification_evidence import capture_checkout
ROOT = Path(__file__).resolve().parents[1]
LIMITS = [
'Screenshots require visual review; automated success is not visual approval.',
'IME, folder chooser, notification clicks and paid provider sessions require manual checks.',
'Native accessibility QA runs only in desktop mode; Xvfb does not certify a compositor.',
]
def interrupted(_signal, _frame):
raise KeyboardInterrupt('termination requested')
def probe(argv):
return subprocess.check_output(argv, cwd=ROOT, text=True, stderr=subprocess.STDOUT,
timeout=30).strip()
def doctor(mode):
errors, versions = [], {}
if mode != 'doctor' and os.geteuid() == 0:
errors.append('run verification as an unprivileged user; filesystem-permission tests cannot run as root')
required = ['cargo', 'rustc', 'rustfmt', 'git', 'pkg-config', 'cc', 'node']
if mode in {'doctor', 'full', 'desktop'}:
required += ['xvfb-run', 'Xvfb', 'xauth', 'dbus-run-session']
for executable in required:
if not shutil.which(executable):
errors.append(f'missing {executable}; run scripts/setup-dev.sh --system')
for package, minimum in [('gtk4', '4.18'), ('libadwaita-1', '1.7'),
('vte-2.91-gtk4', '0.80')]:
if shutil.which('pkg-config'):
try:
probe(['pkg-config', f'--atleast-version={minimum}', package])
versions[package] = probe(['pkg-config', '--modversion', package])
except subprocess.SubprocessError:
errors.append(f'require {package} >= {minimum}; run scripts/setup-dev.sh --system')
if shutil.which('rustc'):
try:
versions['rustc'] = probe(['rustc', '--version'])
version = versions['rustc'].split()[1].split('-')[0]
if tuple(map(int, version.split('.'))) < (1, 92, 0):
errors.append('locked GTK dependencies require Rust >= 1.92')
except (subprocess.SubprocessError, ValueError, IndexError) as error:
errors.append(f'cannot inspect Rust: {error}')
if shutil.which('cargo'):
try:
versions['clippy'] = probe(['cargo', 'clippy', '--version'])
except subprocess.SubprocessError:
errors.append('missing Clippy; run rustup component add clippy')
if mode == 'desktop':
if not (os.environ.get('DISPLAY') or os.environ.get('WAYLAND_DISPLAY')):
errors.append('desktop mode needs a real graphical session')
try:
probe([sys.executable, '-c',
'import gi; gi.require_version("Atspi", "2.0"); from gi.repository import Gio, GLib, Atspi'])
except subprocess.SubprocessError:
errors.append('desktop mode needs Python gi and Atspi bindings')
return errors, versions
class Verification:
def __init__(self, mode, output, before, provenance_error):
self.output = output
self.env = dict(os.environ, CARGO_TARGET_DIR=str(ROOT / 'target'),
SIGNALTTY_QA_LOG_DIR=str(output / 'qa-logs'))
# Existing native helpers resolve binaries under the checkout's target/debug.
# A separate checkout therefore owns its binaries and app sessions.
self.report = dict(schema_version=1, mode=mode, status='running', steps=[], errors=[], limits=LIMITS,
revision=before['revision'] if before else None,
provenance=dict(schema_version=1, before=before, after=None, unchanged=False))
if provenance_error:
self.report['provenance']['error'] = provenance_error
if before:
try:
self.report['dirty'] = bool(probe(['git', 'status', '--porcelain']))
except subprocess.SubprocessError:
self.report['dirty'] = None
def fail(self, category, error):
failed = next((step for step in reversed(self.report['steps']) if step['status'] == 'failed'), {})
log = str(self.output / failed['log']) if failed else str(self.output / 'summary.json')
recovery = {
'environment': 'Read the environment errors; run scripts/setup-dev.sh --system and scripts/verify.sh doctor before rerunning.',
'provenance': 'Inspect Git availability and checkout root; restore source provenance before rerunning scripts/verify.sh full.',
'stale': 'Finish source writes or use an isolated checkout, then run scripts/verify.sh full again.',
'interruption': f'Read {log}; confirm the owned verification processes stopped, then rerun when the interruption is resolved.',
'command': f'Read {log} and qa-logs; diagnose the failing command before one bounded rerun. If it fails again, gather new evidence and revise the hypothesis.',
}[category]
self.report.update(status='failed', failure=dict(category=category, failed_step=failed.get('name'),
log=log, recovery=recovery))
self.report['errors'].append(str(error) or 'interrupted')
def finish_provenance(self):
provenance = self.report['provenance']
try:
after = capture_checkout(ROOT, self.output)
before = provenance['before']
provenance.update(after=after, unchanged=before is not None and all(
before[key] == after[key] for key in ('root', 'content_digest', 'index_digest', 'files')))
if before is not None and not provenance['unchanged'] and self.report['status'] == 'passed':
self.fail('stale', 'checkout inputs changed during verification; successful commands do not prove the current source')
except (ValueError, OSError, subprocess.SubprocessError) as error:
provenance['error'] = str(error)
if self.report['mode'] != 'doctor' and self.report['status'] != 'failed':
self.fail('provenance', f'cannot capture final source provenance: {error}')
def save(self):
(self.output / 'summary.json').write_text(json.dumps(self.report, indent=2) + '\n')
def run(self, name, argv, timeout=900, env=None):
log = self.output / f'{name}.log'
step = dict(name=name, command=list(map(str, argv)), log=log.name, status='running')
self.report['steps'].append(step)
self.save()
print(f'[{name}] {" ".join(map(str, argv))}', flush=True)
started = time.monotonic()
process = None
try:
with log.open('w') as stream:
process = subprocess.Popen(argv, cwd=ROOT, env=self.env | (env or {}),
stdout=stream, stderr=subprocess.STDOUT,
start_new_session=True)
step['exit_code'] = process.wait(timeout=timeout)
if step['exit_code']:
raise RuntimeError(f'{name} failed ({step["exit_code"]}); see {log}')
step['status'] = 'passed'
except BaseException:
step['status'] = 'failed'
if process is not None:
try:
os.killpg(process.pid, signal.SIGTERM)
# QA helpers first stop their separately-sessioned servers/GUI.
# Their bounded socket waits plus child waits can exceed 5 seconds.
process.wait(timeout=30)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
except ProcessLookupError:
pass
if log.exists():
print(log.read_text(errors='replace')[-6000:], file=sys.stderr)
raise
finally:
step['seconds'] = round(time.monotonic() - started, 3)
self.save()
return log
def checks(self, mode):
# Resolve locked dependencies before offline packaging checks on fresh runners.
metadata = self.run('metadata', ['cargo', 'metadata', '--locked', '--format-version', '1'])
self.run('tooling-tests', [sys.executable, '-m', 'unittest', 'discover', '-s', 'scripts/tests', '-v'])
self.run('repository', [str(ROOT / 'scripts/agent.sh'), 'check', '--phase', 'repository'])
self.run('fmt', ['cargo', 'fmt', '--check'])
metadata_json = self.output / 'metadata.json'
metadata_json.write_text('\n'.join(line for line in metadata.read_text().splitlines()
if line.startswith('{')))
self.run('architecture', [sys.executable, 'scripts/check-quality.py', 'architecture', str(metadata_json)])
diagnostics = self.run('clippy', ['cargo', 'clippy', '--workspace', '--all-targets', '--locked',
'--message-format=json'])
# Cargo progress goes to stderr; retain it in the log, extract only JSON records.
records = self.output / 'clippy.jsonl'
records.write_text('\n'.join(line for line in diagnostics.read_text().splitlines()
if line.startswith('{')) + '\n')
self.run('warning-policy', [sys.executable, 'scripts/check-quality.py', 'clippy', str(records)])
# Testkit locates real target/debug executables, not the Cargo test binaries.
self.run('build', ['cargo', 'build', '--workspace', '--locked'])
self.run('workspace-tests', ['cargo', 'test', '--workspace', '--locked'])
if mode == 'fast':
return
self.run('server-qa', [sys.executable, 'scripts/qa-server-edge-cases.py'])
artifacts = self.run('gui-build', ['cargo', 'test', '-p', 'signaltty-gui', '--locked',
'--no-run', '--message-format=json'])
binaries = set()
for line in artifacts.read_text().splitlines():
if line.startswith('{'):
value = json.loads(line)
if value.get('reason') == 'compiler-artifact' and value['profile']['test'] and value.get('executable'):
binaries.add(value['executable'])
count = 0
for index, binary in enumerate(sorted(binaries)):
listing = self.run(f'gui-list-{index}', [binary, '--list', '--ignored', '--format', 'terse'])
names = [line.removesuffix(': test') for line in listing.read_text().splitlines()
if line.endswith(': test')]
for name in names:
count += 1
self.run(f'gtk-{count:02d}', ['xvfb-run', '-a', '-s', '-screen 0 1600x1200x24', 'dbus-run-session', '--',
binary, '--exact', name, '--ignored', '--test-threads=1'],
timeout=120, env={'GTK_A11Y': 'none', 'GDK_BACKEND': 'x11',
'GSK_RENDERER': 'cairo',
'SIGNALTTY_UI_EVIDENCE': str(self.output / 'screenshots')})
if not count:
raise RuntimeError('no ignored GTK tests discovered')
self.report['display_tests'] = count
self.run('refresh-benchmark', ['xvfb-run', '-a', '-s', '-screen 0 1600x1200x24', sys.executable,
'scripts/bench-gui-refresh.py', '--check'],
env={'GTK_A11Y': 'none', 'GDK_BACKEND': 'x11', 'GSK_RENDERER': 'cairo'})
if mode == 'desktop':
self.run('native-accessibility', [sys.executable, 'scripts/qa-gui-reliability.py'])
def main():
signal.signal(signal.SIGTERM, interrupted)
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('mode', choices=['doctor', 'fast', 'full', 'desktop'])
parser.add_argument('--output', type=Path, help='new artifact directory (must not exist)')
args = parser.parse_args()
if args.output:
output = args.output.resolve()
if output.exists():
parser.error(f'evidence directory already exists: {output}')
else:
output = ROOT / 'target' / 'verification'
before, provenance_error = None, None
try:
before = capture_checkout(ROOT, output)
except (ValueError, OSError, subprocess.SubprocessError) as error:
if isinstance(error, ValueError) and 'evidence output must be ignored' in str(error):
parser.error(str(error))
provenance_error = str(error)
if args.output and ROOT in output.parents:
parser.error(f'cannot establish that evidence output is ignored: {error}')
if args.output:
try:
output.mkdir(parents=True, exist_ok=False)
except FileExistsError:
parser.error(f'evidence directory already exists: {output}')
else:
base = ROOT / 'target' / 'verification'
base.mkdir(parents=True, exist_ok=True)
output = Path(tempfile.mkdtemp(prefix=f'{args.mode}-', dir=base))
print(f'Evidence: {output}', flush=True)
run = Verification(args.mode, output, before, provenance_error)
category = 'environment'
try:
errors, versions = doctor(args.mode)
run.report.update(errors=errors, versions=versions)
if errors:
raise RuntimeError('environment diagnosis failed: ' + '; '.join(errors))
if args.mode != 'doctor':
category = 'provenance'
if before is None:
raise RuntimeError(f'source provenance unavailable: {provenance_error}')
category = 'command'
run.checks(args.mode)
run.report['status'] = 'passed'
except (Exception, KeyboardInterrupt) as error:
if isinstance(error, (KeyboardInterrupt, subprocess.TimeoutExpired)):
category = 'interruption'
run.fail(category, error)
print(f'FAIL: {error}', file=sys.stderr)
finally:
run.finish_provenance()
run.save()
return run.report['status'] != 'passed'
if __name__ == '__main__':
raise SystemExit(main())