chore(jarca-ledger): finalize approved batch docs, dashboard, and CI smoke updates
This commit is contained in:
Jarca
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CI wrapper for the Streamlit dashboard HTTP smoke gate.
|
||||
|
||||
Creates a synthetic fixture DB, starts Streamlit against it, waits for HTTP 200,
|
||||
and always attempts process cleanup. Output is intentionally status-only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.ci_fixture_db import create_ci_fixture_db
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Run Streamlit dashboard HTTP smoke against a synthetic CI fixture DB.")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="Loopback host to bind")
|
||||
parser.add_argument("--port", type=int, default=18501, help="Deterministic smoke port")
|
||||
parser.add_argument("--timeout", type=int, default=45, help="Readiness timeout in seconds")
|
||||
parser.add_argument("--page", default="요약", help="Dashboard page query param to request")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def wait_for_http(url: str, timeout_seconds: int) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
attempts = 0
|
||||
last_error = "not_started"
|
||||
while time.monotonic() < deadline:
|
||||
attempts += 1
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return {"ready": True, "status": response.status, "attempts": attempts}
|
||||
last_error = f"HTTP {response.status}"
|
||||
except Exception as exc: # noqa: BLE001 - compact readiness diagnostic
|
||||
last_error = exc.__class__.__name__
|
||||
time.sleep(1)
|
||||
return {"ready": False, "attempts": attempts, "last_error": last_error}
|
||||
|
||||
|
||||
def terminate_process(proc: subprocess.Popen[str], timeout: int = 10) -> dict[str, Any]:
|
||||
if proc.poll() is not None:
|
||||
return {"cleanup": "already_exited", "returncode": proc.returncode}
|
||||
try:
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
proc.wait(timeout=timeout)
|
||||
return {"cleanup": "terminated", "returncode": proc.returncode}
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=timeout)
|
||||
return {"cleanup": "killed", "returncode": proc.returncode}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
smoke_url = f"http://{args.host}:{args.port}/?page={urllib.parse.quote(args.page)}"
|
||||
proc: subprocess.Popen[str] | None = None
|
||||
with tempfile.TemporaryDirectory(prefix="jarca-ledger-ci-http-") as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
fixture_path = tmp_path / "fixture.sqlite3"
|
||||
try:
|
||||
fixture = create_ci_fixture_db(fixture_path)
|
||||
env = os.environ.copy()
|
||||
env["JARCA_LEDGER_DB_PATH"] = str(fixture_path)
|
||||
env["STREAMLIT_BROWSER_GATHER_USAGE_STATS"] = "false"
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"streamlit",
|
||||
"run",
|
||||
"dashboard/app.py",
|
||||
"--server.address",
|
||||
args.host,
|
||||
"--server.port",
|
||||
str(args.port),
|
||||
"--server.headless",
|
||||
"true",
|
||||
"--browser.gatherUsageStats",
|
||||
"false",
|
||||
]
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
cwd=Path(__file__).resolve().parents[1],
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
readiness = wait_for_http(smoke_url, args.timeout)
|
||||
if not readiness["ready"]:
|
||||
cleanup = terminate_process(proc)
|
||||
print(json.dumps({"success": False, "gate": "dashboard-http-smoke", "fixture": "synthetic", "readiness": readiness, "cleanup": cleanup}, ensure_ascii=False))
|
||||
return 10
|
||||
cleanup = terminate_process(proc)
|
||||
success = cleanup.get("cleanup") in {"terminated", "already_exited", "killed"}
|
||||
print(json.dumps({
|
||||
"success": success,
|
||||
"gate": "dashboard-http-smoke",
|
||||
"fixture": "synthetic",
|
||||
"fixture_integrity": fixture.get("integrity_check"),
|
||||
"http_status": readiness.get("status"),
|
||||
"attempts": readiness.get("attempts"),
|
||||
"cleanup": cleanup.get("cleanup"),
|
||||
}, ensure_ascii=False, indent=2))
|
||||
return 0 if success else 11
|
||||
except Exception as exc: # noqa: BLE001 - deterministic wrapper failure
|
||||
cleanup = terminate_process(proc) if proc is not None else {"cleanup": "not_started"}
|
||||
print(json.dumps({"success": False, "gate": "dashboard-http-smoke", "error_code": "wrapper_error", "error": exc.__class__.__name__, "cleanup": cleanup.get("cleanup")}, ensure_ascii=False))
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CI wrapper for Jarca Ledger DB restore/safety smoke.
|
||||
|
||||
The wrapper builds a synthetic fixture DB under a temporary directory, copies it
|
||||
through the existing restore smoke script, and emits only status-level evidence.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from jarca_ledger_mcp.db import DEFAULT_DB_PATH
|
||||
from scripts.ci_fixture_db import create_ci_fixture_db
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Run DB safety smoke with synthetic temp DBs only.")
|
||||
parser.add_argument("--timeout", type=int, default=45, help="Subprocess timeout in seconds")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def parse_json_output(output: str) -> dict[str, Any]:
|
||||
try:
|
||||
return json.loads(output)
|
||||
except json.JSONDecodeError:
|
||||
return {"success": False, "error_code": "invalid_json_output"}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
repo = Path(__file__).resolve().parents[1]
|
||||
with tempfile.TemporaryDirectory(prefix="jarca-ledger-ci-db-") as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
source = tmp_path / "fixture.sqlite3"
|
||||
backup = tmp_path / "backup.sqlite3"
|
||||
target = tmp_path / "restored.sqlite3"
|
||||
try:
|
||||
fixture = create_ci_fixture_db(source)
|
||||
backup.write_bytes(source.read_bytes())
|
||||
if any(path.resolve() == DEFAULT_DB_PATH.resolve() for path in [source, backup, target]):
|
||||
print(json.dumps({"success": False, "gate": "db-safety-smoke", "error_code": "operating_db_path_refused"}, ensure_ascii=False))
|
||||
return 20
|
||||
env = os.environ.copy()
|
||||
env["JARCA_LEDGER_DB_PATH"] = str(source)
|
||||
cmd = [sys.executable, "scripts/ledger_restore_smoke.py", "--backup", str(backup), "--target", str(target)]
|
||||
run = subprocess.run(cmd, cwd=repo, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=args.timeout, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
print(json.dumps({"success": False, "gate": "db-safety-smoke", "error_code": "timeout", "timeout_seconds": args.timeout}, ensure_ascii=False))
|
||||
return 21
|
||||
except Exception as exc: # noqa: BLE001 - deterministic wrapper failure
|
||||
print(json.dumps({"success": False, "gate": "db-safety-smoke", "error_code": "wrapper_error", "error": exc.__class__.__name__}, ensure_ascii=False))
|
||||
return 1
|
||||
|
||||
smoke = parse_json_output(run.stdout)
|
||||
success = run.returncode == 0 and bool(smoke.get("success")) and target.exists()
|
||||
result = {
|
||||
"success": success,
|
||||
"gate": "db-safety-smoke",
|
||||
"fixture": "synthetic",
|
||||
"fixture_integrity": fixture.get("integrity_check"),
|
||||
"restore_smoke_exit_code": run.returncode,
|
||||
"restored_exists": target.exists(),
|
||||
"restored_integrity": smoke.get("target", {}).get("integrity_check"),
|
||||
"health_success": smoke.get("health_check", {}).get("success"),
|
||||
"operating_db_used": False,
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if success else 22
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+118
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synthetic fixture DB builder for CI/AppTest/smoke checks.
|
||||
|
||||
The fixture intentionally contains only synthetic identifiers and summary data.
|
||||
It must never point at the operating ledger database.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from jarca_ledger_mcp import service
|
||||
from jarca_ledger_mcp.db import DEFAULT_DB_PATH, init_db
|
||||
|
||||
|
||||
def assert_ci_fixture_db_path(path: Path) -> Path:
|
||||
resolved = path.expanduser().resolve()
|
||||
if resolved == DEFAULT_DB_PATH.resolve():
|
||||
raise ValueError("refusing to use the operating ledger DB as a CI fixture")
|
||||
if resolved.exists() and resolved.is_dir():
|
||||
raise ValueError(f"fixture DB path is a directory: {resolved}")
|
||||
allowed_names = {"fixture.sqlite3", "ci-ledger-fixture.sqlite3", "ledger-fixture.sqlite3"}
|
||||
if resolved.name not in allowed_names:
|
||||
raise ValueError(f"CI fixture DB filename must be one of {sorted(allowed_names)}: {resolved.name}")
|
||||
return resolved
|
||||
|
||||
|
||||
def create_ci_fixture_db(db_path: str | Path) -> dict[str, Any]:
|
||||
path = assert_ci_fixture_db_path(Path(db_path))
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if path.exists():
|
||||
path.unlink()
|
||||
init_db(str(path))
|
||||
service.seed_default_ledger_categories(db_path=str(path))
|
||||
service.seed_default_category_rules(db_path=str(path))
|
||||
|
||||
account = service.create_ledger_account(
|
||||
"CI Synthetic Checking",
|
||||
institution="CI Synthetic Bank",
|
||||
holder_name="CI Synthetic Holder",
|
||||
last_balance=1_250_000,
|
||||
last_synced_at="2026-06-30T23:59:00+09:00",
|
||||
statement_status="partial_import",
|
||||
liquidity_bucket="cash",
|
||||
memo="ci synthetic fixture account",
|
||||
db_path=str(path),
|
||||
)
|
||||
if not account.get("success"):
|
||||
raise RuntimeError(f"fixture account seed failed: {account.get('error_code')}")
|
||||
account_id = account["account"]["id"]
|
||||
|
||||
for account_spec in [
|
||||
dict(name="CI Synthetic Investment", type="investment", institution="CI Synthetic Securities", last_balance=5_000_000, liquidity_bucket="investment"),
|
||||
dict(name="CI Synthetic Loan", type="loan", institution="CI Synthetic Bank", last_balance=2_000_000, liquidity_bucket="loan"),
|
||||
dict(name="CI Synthetic Manual Asset", type="other", institution="Manual", liquidity_bucket="real_estate"),
|
||||
]:
|
||||
result = service.create_ledger_account(
|
||||
last_synced_at="2026-06-30T23:59:00+09:00",
|
||||
memo="ci synthetic fixture account",
|
||||
db_path=str(path),
|
||||
**account_spec,
|
||||
)
|
||||
if not result.get("success"):
|
||||
raise RuntimeError(f"fixture account seed failed: {result.get('error_code')}")
|
||||
|
||||
budget = service.create_ledger_budget("2026-06", "식비", 50_000, memo="ci synthetic budget", db_path=str(path))
|
||||
if not budget.get("success"):
|
||||
raise RuntimeError(f"fixture budget seed failed: {budget.get('error_code')}")
|
||||
|
||||
tx_specs: list[dict[str, Any]] = [
|
||||
dict(transaction_date="2026-06-25", type="income", amount=2_000_000, merchant="CI_SYNTHETIC_SALARY", category="수입", subcategory="급여", source="manual", memo="ci synthetic income"),
|
||||
dict(transaction_date="2026-06-26", type="expense", amount=35_000, merchant="CI_SYNTHETIC_DINING", category="식비", subcategory="외식", source="manual", memo="ci synthetic expense"),
|
||||
dict(transaction_date="2026-06-27", type="expense", amount=18_500, merchant="CI_SYNTHETIC_MART", category="", subcategory="", source="manual", memo="ci synthetic uncategorized", raw_summary="ci synthetic grocery"),
|
||||
dict(transaction_date="2026-06-28", type="expense", amount=4_800, merchant="CI_SYNTHETIC_CAFE", category="", subcategory="", source="manual", memo="ci synthetic uncategorized", raw_summary="ci synthetic cafe"),
|
||||
dict(transaction_date="2026-06-29", type="expense", amount=120_000, merchant="CI_SYNTHETIC_TRANSFER", category="이체", subcategory="확인필요", source="kb_bank_pdf", counterparty="CI_SYNTHETIC_TOSS", memo="cross_transfer_review: ci synthetic", raw_summary="ci synthetic transfer", balance_after=1_250_000),
|
||||
]
|
||||
transaction_ids: list[str] = []
|
||||
for index, spec in enumerate(tx_specs, start=1):
|
||||
result = service.create_ledger_transaction(
|
||||
account_id=account_id,
|
||||
external_id=f"ci-fixture-{index}",
|
||||
db_path=str(path),
|
||||
**spec,
|
||||
)
|
||||
if not result.get("success"):
|
||||
raise RuntimeError(f"fixture transaction seed failed: {result.get('error_code')}")
|
||||
transaction_ids.append(result["transaction"]["id"])
|
||||
|
||||
status = service.get_ledger_db_status(str(path), include_hash=False, include_integrity=True)
|
||||
if not status.get("success") or status.get("integrity_check") != "ok":
|
||||
raise RuntimeError(f"fixture health check failed: {status.get('error_code')}")
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"fixture_db": str(path),
|
||||
"account_count": 4,
|
||||
"transaction_count": len(transaction_ids),
|
||||
"integrity_check": status.get("integrity_check"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Create a synthetic Jarca Ledger CI fixture DB.")
|
||||
parser.add_argument("--db", required=True, help="Output fixture path, e.g. $RUNNER_TEMP/fixture.sqlite3")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
result = create_ci_fixture_db(args.db)
|
||||
except Exception as exc: # noqa: BLE001 - CLI wrapper reports deterministic failure
|
||||
print(json.dumps({"success": False, "error_code": "fixture_seed_failed", "error": str(exc)}, ensure_ascii=False))
|
||||
return 1
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+353
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture local Streamlit browser evidence with Chrome and a synthetic DB.
|
||||
|
||||
This script is intentionally local-only. It creates a synthetic fixture database,
|
||||
starts Streamlit against that database, and uses an installed Chrome binary to
|
||||
save page screenshots and DOM snapshots. It must not target the operating ledger
|
||||
DB or real financial statement files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import websockets
|
||||
|
||||
from jarca_ledger_mcp import service
|
||||
from jarca_ledger_mcp.db import init_db
|
||||
|
||||
DEFAULT_PAGES = ["분류 규칙", "확인필요 큐", "Import"]
|
||||
DEFAULT_CHROME = "/usr/bin/google-chrome-stable"
|
||||
DEFAULT_STREAMLIT = Path(".venv/bin/streamlit")
|
||||
DEFAULT_OUTPUT_ROOT = Path("docs/batches/bat-004_20260629_browser-streamlit-e2e-evidence/evidence")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Capture Streamlit browser screenshots and DOM snapshots with a synthetic DB.")
|
||||
parser.add_argument("--chrome", default=os.getenv("CHROME_BIN", DEFAULT_CHROME), help="Chrome executable path")
|
||||
parser.add_argument("--streamlit", default=str(DEFAULT_STREAMLIT), help="Streamlit executable path")
|
||||
parser.add_argument("--port", type=int, default=18501, help="Local Streamlit port")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="Local Streamlit host")
|
||||
parser.add_argument("--output-dir", default="", help="Evidence output directory. Defaults under docs/batches/BAT-004/evidence/run-<timestamp>.")
|
||||
parser.add_argument("--page", action="append", dest="pages", help="Page to capture. Repeatable. Defaults to core E2E evidence pages.")
|
||||
parser.add_argument("--keep-server", action="store_true", help="Leave Streamlit running after capture for manual inspection.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def slugify_page(name: str) -> str:
|
||||
mapping = {
|
||||
"분류 규칙": "rules",
|
||||
"확인필요 큐": "transfer-review",
|
||||
"Import": "import",
|
||||
"계좌": "accounts",
|
||||
"요약": "overview",
|
||||
"거래내역": "transactions",
|
||||
"미분류": "uncategorized",
|
||||
"카테고리": "categories",
|
||||
}
|
||||
return mapping.get(name, urllib.parse.quote(name, safe="").replace("%", "").lower() or "page")
|
||||
|
||||
|
||||
def require_executable(path: str, label: str) -> str:
|
||||
resolved = shutil.which(path) if os.path.sep not in path else path
|
||||
if not resolved or not Path(resolved).exists():
|
||||
raise SystemExit(f"{label} executable not found: {path}")
|
||||
return resolved
|
||||
|
||||
|
||||
def assert_synthetic_db_path(path: Path) -> None:
|
||||
resolved = path.expanduser().resolve()
|
||||
operating = Path.home() / ".local" / "share" / "jarca" / "ledger.sqlite3"
|
||||
if resolved == operating.resolve():
|
||||
raise SystemExit("Refusing to use the operating ledger DB for browser evidence")
|
||||
if not str(resolved).endswith("fixture.sqlite3"):
|
||||
raise SystemExit(f"Synthetic evidence DB must end with fixture.sqlite3: {resolved}")
|
||||
|
||||
|
||||
def seed_fixture_db(db_path: Path) -> dict[str, Any]:
|
||||
assert_synthetic_db_path(db_path)
|
||||
if db_path.exists():
|
||||
db_path.unlink()
|
||||
init_db(str(db_path))
|
||||
service.seed_default_ledger_categories(db_path=str(db_path))
|
||||
service.seed_default_category_rules(db_path=str(db_path))
|
||||
|
||||
account = service.create_ledger_account(
|
||||
"Synthetic Evidence Checking",
|
||||
institution="Synthetic Bank",
|
||||
account_number_masked="SYNTH-0000",
|
||||
holder_name="Synthetic Holder",
|
||||
last_balance=1_250_000,
|
||||
last_synced_at="2026-06-30T23:59:00+09:00",
|
||||
statement_status="partial_import",
|
||||
liquidity_bucket="cash",
|
||||
memo="browser evidence synthetic fixture",
|
||||
db_path=str(db_path),
|
||||
)["account"]
|
||||
account_id = account["id"]
|
||||
service.create_ledger_account(
|
||||
"Synthetic Investment Account",
|
||||
type="investment",
|
||||
institution="Synthetic Securities",
|
||||
last_balance=5_000_000,
|
||||
last_synced_at="2026-06-30T23:59:00+09:00",
|
||||
liquidity_bucket="investment",
|
||||
memo="browser evidence investment asset",
|
||||
db_path=str(db_path),
|
||||
)
|
||||
service.create_ledger_account(
|
||||
"Synthetic Mortgage Balance",
|
||||
type="loan",
|
||||
institution="Synthetic Bank",
|
||||
last_balance=2_000_000,
|
||||
last_synced_at="2026-06-30T23:59:00+09:00",
|
||||
liquidity_bucket="loan",
|
||||
memo="browser evidence liability",
|
||||
db_path=str(db_path),
|
||||
)
|
||||
service.create_ledger_account(
|
||||
"Synthetic Appraisal Pending",
|
||||
type="other",
|
||||
institution="Manual",
|
||||
liquidity_bucket="real_estate",
|
||||
memo="browser evidence missing last_balance warning",
|
||||
db_path=str(db_path),
|
||||
)
|
||||
service.create_ledger_budget("2026-06", "식비", 20_000, memo="browser evidence budget", db_path=str(db_path))
|
||||
|
||||
tx_specs = [
|
||||
dict(transaction_date="2026-06-25", type="income", amount=2_000_000, merchant="SYNTHETIC_SALARY", category="수입", subcategory="급여", source="manual", memo="browser evidence income"),
|
||||
dict(transaction_date="2026-06-26", type="expense", amount=35_000, merchant="SYNTHETIC_DINING", category="식비", subcategory="외식", source="manual", memo="browser evidence expense"),
|
||||
dict(transaction_date="2026-06-27", type="expense", amount=18_500, merchant="SYNTHETIC_MART", category="", subcategory="", source="manual", memo="browser evidence uncategorized", raw_summary="synthetic grocery"),
|
||||
dict(transaction_date="2026-06-28", type="expense", amount=4_800, merchant="SYNTHETIC_CAFE", category="", subcategory="", source="manual", memo="browser evidence uncategorized", raw_summary="synthetic cafe"),
|
||||
dict(transaction_date="2026-06-29", type="transfer", amount=120_000, merchant="SYNTHETIC_TRANSFER", category="이체", subcategory="확인필요", source="kb_bank_pdf", counterparty="SYNTHETIC_TOSS", memo="cross_transfer_review: synthetic browser evidence", raw_summary="synthetic transfer", balance_after=1_250_000),
|
||||
]
|
||||
transactions = []
|
||||
for index, spec in enumerate(tx_specs, start=1):
|
||||
result = service.create_ledger_transaction(account_id=account_id, external_id=f"browser-evidence-{index}", db_path=str(db_path), **spec)
|
||||
if not result.get("success"):
|
||||
raise SystemExit(f"Fixture transaction seed failed: {result}")
|
||||
transactions.append(result["transaction"]["id"])
|
||||
|
||||
return {"account_id": account_id, "transaction_ids": transactions, "db_path": str(db_path)}
|
||||
|
||||
|
||||
def wait_for_http(url: str, timeout_seconds: int = 45) -> None:
|
||||
deadline = time.time() + timeout_seconds
|
||||
last_error: str | None = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
last_error = f"HTTP {response.status}"
|
||||
except Exception as exc: # noqa: BLE001 - diagnostics returned in manifest
|
||||
last_error = str(exc)
|
||||
time.sleep(1)
|
||||
raise SystemExit(f"Streamlit did not become ready at {url}: {last_error}")
|
||||
|
||||
|
||||
def read_json_url(url: str, timeout: int = 2) -> Any:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def wait_for_devtools(port: int, timeout_seconds: int = 15) -> str:
|
||||
deadline = time.time() + timeout_seconds
|
||||
last_error: str | None = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
targets = read_json_url(f"http://127.0.0.1:{port}/json/list")
|
||||
for target in targets:
|
||||
if target.get("type") == "page" and target.get("webSocketDebuggerUrl"):
|
||||
return target["webSocketDebuggerUrl"]
|
||||
last_error = "no page target"
|
||||
except Exception as exc: # noqa: BLE001 - Chrome startup diagnostics
|
||||
last_error = str(exc)
|
||||
time.sleep(0.25)
|
||||
raise SystemExit(f"Chrome DevTools endpoint did not become ready on port {port}: {last_error}")
|
||||
|
||||
|
||||
class CdpClient:
|
||||
def __init__(self, ws: websockets.ClientConnection) -> None:
|
||||
self.ws = ws
|
||||
self.next_id = 1
|
||||
|
||||
async def call(self, method: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
command_id = self.next_id
|
||||
self.next_id += 1
|
||||
await self.ws.send(json.dumps({"id": command_id, "method": method, "params": params or {}}))
|
||||
while True:
|
||||
raw = await self.ws.recv()
|
||||
payload = json.loads(raw)
|
||||
if payload.get("id") == command_id:
|
||||
if "error" in payload:
|
||||
raise RuntimeError(f"CDP {method} failed: {payload['error']}")
|
||||
return payload.get("result", {})
|
||||
|
||||
|
||||
async def cdp_capture(ws_url: str, url: str, expected_text: str, screenshot: Path, snapshot: Path) -> dict[str, Any]:
|
||||
async with websockets.connect(ws_url, max_size=16 * 1024 * 1024) as ws:
|
||||
cdp = CdpClient(ws)
|
||||
await cdp.call("Page.enable")
|
||||
await cdp.call("Runtime.enable")
|
||||
await cdp.call(
|
||||
"Emulation.setDeviceMetricsOverride",
|
||||
{"width": 1440, "height": 1400, "deviceScaleFactor": 1, "mobile": False},
|
||||
)
|
||||
await cdp.call("Page.navigate", {"url": url})
|
||||
|
||||
inner_text = ""
|
||||
deadline = time.time() + 45
|
||||
while time.time() < deadline:
|
||||
result = await cdp.call(
|
||||
"Runtime.evaluate",
|
||||
{"expression": "document.body ? document.body.innerText : ''", "returnByValue": True},
|
||||
)
|
||||
inner_text = result.get("result", {}).get("value", "") or ""
|
||||
if expected_text in inner_text:
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
if expected_text not in inner_text:
|
||||
raise SystemExit(f"Browser page did not render expected text {expected_text!r}; last text starts with {inner_text[:200]!r}")
|
||||
|
||||
html_result = await cdp.call(
|
||||
"Runtime.evaluate",
|
||||
{"expression": "document.documentElement.outerHTML", "returnByValue": True},
|
||||
)
|
||||
snapshot.write_text(html_result.get("result", {}).get("value", ""), encoding="utf-8")
|
||||
image_result = await cdp.call("Page.captureScreenshot", {"format": "png", "fromSurface": True, "captureBeyondViewport": True})
|
||||
screenshot.write_bytes(base64.b64decode(image_result["data"]))
|
||||
return {"inner_text_head": inner_text[:500]}
|
||||
|
||||
|
||||
def capture_page(chrome: str, base_url: str, page: str, output_dir: Path, devtools_port: int) -> dict[str, Any]:
|
||||
slug = slugify_page(page)
|
||||
url = f"{base_url}/?page={urllib.parse.quote(page)}"
|
||||
screenshot = output_dir / f"{slug}.png"
|
||||
snapshot = output_dir / f"{slug}.dom.html"
|
||||
|
||||
profile_dir = output_dir / f"chrome-profile-{slug}"
|
||||
chrome_cmd = [
|
||||
chrome,
|
||||
"--headless=new",
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--disable-software-rasterizer",
|
||||
"--hide-scrollbars",
|
||||
f"--remote-debugging-port={devtools_port}",
|
||||
f"--user-data-dir={profile_dir}",
|
||||
"--window-size=1440,1400",
|
||||
"about:blank",
|
||||
]
|
||||
chrome_proc = subprocess.Popen(chrome_cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
try:
|
||||
ws_url = wait_for_devtools(devtools_port)
|
||||
cdp_info = asyncio.run(cdp_capture(ws_url, url, page, screenshot, snapshot))
|
||||
finally:
|
||||
if chrome_proc.poll() is None:
|
||||
chrome_proc.terminate()
|
||||
try:
|
||||
chrome_proc.wait(timeout=8)
|
||||
except subprocess.TimeoutExpired:
|
||||
chrome_proc.kill()
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
|
||||
if not screenshot.exists() or screenshot.stat().st_size <= 0:
|
||||
raise SystemExit(f"Screenshot was not created for {page}: {screenshot}")
|
||||
if page not in snapshot.read_text(encoding="utf-8", errors="ignore"):
|
||||
raise SystemExit(f"DOM snapshot does not contain expected page title {page!r}: {snapshot}")
|
||||
|
||||
return {
|
||||
"page": page,
|
||||
"url": url,
|
||||
"screenshot": str(screenshot),
|
||||
"screenshot_bytes": screenshot.stat().st_size,
|
||||
"snapshot": str(snapshot),
|
||||
"snapshot_bytes": snapshot.stat().st_size,
|
||||
"devtools_port": devtools_port,
|
||||
"inner_text_head": cdp_info.get("inner_text_head", ""),
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
chrome = require_executable(args.chrome, "Chrome")
|
||||
streamlit = require_executable(args.streamlit, "Streamlit")
|
||||
run_id = datetime.now().strftime("run-%Y%m%d-%H%M%S")
|
||||
output_dir = Path(args.output_dir) if args.output_dir else DEFAULT_OUTPUT_ROOT / run_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
db_path = output_dir / "fixture.sqlite3"
|
||||
fixture = seed_fixture_db(db_path)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["JARCA_LEDGER_DB_PATH"] = str(db_path.resolve())
|
||||
env["STREAMLIT_BROWSER_GATHER_USAGE_STATS"] = "false"
|
||||
base_url = f"http://{args.host}:{args.port}"
|
||||
server_cmd = [
|
||||
streamlit,
|
||||
"run",
|
||||
"dashboard/app.py",
|
||||
"--server.address",
|
||||
args.host,
|
||||
"--server.port",
|
||||
str(args.port),
|
||||
"--server.headless",
|
||||
"true",
|
||||
"--browser.gatherUsageStats",
|
||||
"false",
|
||||
]
|
||||
|
||||
server = subprocess.Popen(server_cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, start_new_session=True)
|
||||
manifest: dict[str, Any] = {
|
||||
"success": False,
|
||||
"base_url": base_url,
|
||||
"output_dir": str(output_dir.resolve()),
|
||||
"fixture": fixture,
|
||||
"chrome": chrome,
|
||||
"streamlit": streamlit,
|
||||
"pages": [],
|
||||
"server_command": server_cmd,
|
||||
}
|
||||
try:
|
||||
wait_for_http(base_url)
|
||||
for index, page in enumerate(args.pages or DEFAULT_PAGES):
|
||||
manifest["pages"].append(capture_page(chrome, base_url, page, output_dir, devtools_port=19222 + index))
|
||||
manifest["success"] = True
|
||||
print(json.dumps(manifest, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
finally:
|
||||
log_tail = ""
|
||||
if server.stdout is not None:
|
||||
try:
|
||||
time.sleep(0.5)
|
||||
if server.poll() is not None:
|
||||
log_tail = server.stdout.read()[-4000:]
|
||||
except Exception:
|
||||
log_tail = ""
|
||||
manifest["server_log_tail"] = log_tail
|
||||
(output_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
if not args.keep_server and server.poll() is None:
|
||||
os.killpg(server.pid, signal.SIGTERM)
|
||||
try:
|
||||
server.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(server.pid, signal.SIGKILL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user