diff --git a/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/implementation.md b/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/implementation.md index 045fa64..aa5159f 100644 --- a/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/implementation.md +++ b/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/implementation.md @@ -1,16 +1,123 @@ # BAT-017 Implementation Evidence -- Status: not-started +- Status: implemented-awaiting-qc - Created At (KST): 2026-07-08 11:33 KST +- Last Updated (KST): 2026-07-09T13:16:10+09:00 ## Implementation Summary -Pending. +### Slice 01 — provenance foundation, import preview/write sanitization +- Added a shared provenance policy module for source-hash labeling, allowlisted raw-payload JSON serialization, and CSV preview column filtering. +- Stopped new import preview/import write paths from exposing or persisting full local source paths for: + - KB Bank PDF + - Toss Bank PDF + - Toss Securities PDF + - SeoulPay XLSX + - CSV preview +- Changed `import_batches.filename` new writes to `masked-basename#source_hash_prefix`. +- Minimized `transactions.raw_payload` for current supported import sources through explicit allowlists. +- Added targeted regression tests for path redaction and forbidden-field non-leak assertions. ## Changed Files +- `jarca_ledger_mcp/application/imports_service.py` +- `jarca_ledger_mcp/kb_bank_pdf.py` +- `jarca_ledger_mcp/seoulpay_xlsx.py` +- `jarca_ledger_mcp/toss_bank_pdf.py` +- `jarca_ledger_mcp/toss_securities_pdf.py` +- `jarca_ledger_mcp/provenance_policy.py` +- `tests/test_import_provenance_policy.py` -Pending. +## Execution Evidence +- `python -m pytest -q tests/test_import_provenance_policy.py` + - result: pass (`.....`) +- `python -m ruff check jarca_ledger_mcp/application/imports_service.py jarca_ledger_mcp/kb_bank_pdf.py jarca_ledger_mcp/seoulpay_xlsx.py jarca_ledger_mcp/toss_bank_pdf.py jarca_ledger_mcp/toss_securities_pdf.py jarca_ledger_mcp/provenance_policy.py tests/test_import_provenance_policy.py` + - result: pass (`All checks passed!`) +- `pytest -q tests/test_import_provenance_policy.py tests/test_kb_bank_pdf.py tests/test_seoulpay_xlsx.py tests/test_toss_securities_pdf.py` + - result: pass via `dev-impl` +- `pytest -q tests/test_kb_bank_pdf.py tests/test_seoulpay_xlsx.py tests/test_toss_securities_pdf.py tests/test_service.py` + - result: pass via `dev-impl` +- `pytest -q tests/test_import_provenance_policy.py tests/test_kb_bank_pdf.py tests/test_seoulpay_xlsx.py tests/test_toss_securities_pdf.py tests/test_service.py` + - result: pass via `dev-impl` +- `pytest -q` + - result: pass via `dev-impl` + +## Evidence Notes +- Targeted regression confirms preview responses no longer emit full absolute paths. +- CSV preview regression confirms arbitrary raw columns such as `secret_note` and `local_path` are suppressed by default. +- Import persistence regression confirms `import_batches.filename` stores masked basename plus source-hash prefix rather than full paths. +- Import persistence regression confirms `transactions.raw_payload` stores only the source-specific allowlisted minimum fields for the implemented sources. + +## Residual Risks / Follow-up +- Slice 01 does not yet complete the MCP provenance exposure contract (`minimal` / `operator-summary` / `debug-gated`). +- Import/read surfaces beyond the new-write preview/import boundary still need review for finance-agent privacy alignment. +- Baseline docs still need BAT-017-specific updates once the exposure contract and final runtime boundary are settled. +- Existing operating DB sanitization remains intentionally out of scope and must stay in the separate follow-up Todo. ## Notes +- Implementation must not use Kanban unless the user explicitly re-allows it. -Implementation must not use Kanban unless the user explicitly re-allows it. +### Slice 02A — provenance exposure helper + tests +- Added explicit provenance exposure constants and helper APIs in `jarca_ledger_mcp/provenance_policy.py` for `minimal`, `operator-summary`, and `debug-gated`. +- Added sanitizer behavior that keeps normalized provenance fields while suppressing full paths, raw document/raw text dumps, passwords, tokens, secrets, and non-allowlisted raw payload keys. +- Added alias helpers `sanitize_provenance_envelope(...)` and `sanitize_provenance_for_exposure(...)` for bounded downstream adoption. +- Added helper-level regression coverage in `tests/test_import_provenance_policy.py` for exposure-level behavior and forbidden-field suppression. + +## Additional Changed Files (Slice 02A) +- `jarca_ledger_mcp/provenance_policy.py` +- `tests/test_import_provenance_policy.py` + +## Additional Execution Evidence (Slice 02A) +- Supervised impl child result + - artifact: `.stagepilot/worker-supervision/impl-20260709T030219Z/final-result.json` + - result: `done` (`child_exit_code=0`) +- `python -m pytest -q tests/test_import_provenance_policy.py` + - result: pass (`7 passed`) +- `python -m py_compile jarca_ledger_mcp/provenance_policy.py tests/test_import_provenance_policy.py` + - result: pass +- `python -m ruff check jarca_ledger_mcp/provenance_policy.py tests/test_import_provenance_policy.py` + - result: initially blocked in runner environment (`No module named ruff` from current Python) +- Lead follow-up verification after environment check: + - `. .venv/bin/activate && python -m ruff check jarca_ledger_mcp/provenance_policy.py tests/test_import_provenance_policy.py` + - result: pass (`All checks passed!`) + - `. .venv/bin/activate && python -m pytest -q tests/test_import_provenance_policy.py` + - result: pass (`7 passed`) + - `. .venv/bin/activate && python -m py_compile jarca_ledger_mcp/provenance_policy.py tests/test_import_provenance_policy.py` + - result: pass + - `. .venv/bin/activate && python -m ruff check .` + - result: pass (`All checks passed!`) + - `. .venv/bin/activate && python -m pytest -q` + - result: pass (`55 passed, 4 skipped`) + +## Slice 02B — import batch read-surface wiring + baseline docs +- Wired `list_ledger_import_batches(...)` through the BAT-017 provenance sanitizer so default responses now return `minimal` provenance and scrub the raw `memo` field from user-facing payloads. +- Added bounded operator-summary support for import batch listing so masked filename, source-hash prefix, sheet reference, and validation summary are available only when explicitly requested. +- Added MCP adapter pass-through for `provenance_exposure` on `list_ledger_import_batches(...)`. +- Added explicit `debug_provenance_gate=True` enforcement before `debug-gated` provenance can be requested from the import batch listing surface. +- Extended provenance regression coverage for: + - default/minimal import batch responses + - explicit operator-summary import batch responses + - debug-gated access denial without the explicit gate and allow-path with the gate + - MCP adapter pass-through of the exposure level +- Updated baseline docs (`docs/data-model.md`, `docs/project-structure.md`, `docs/runtime-flows.md`) to record the provenance envelope boundary, `raw_payload=minimize`, masked filename policy, and read-time exposure split. + +## Additional Changed Files (Slice 02B) +- `docs/data-model.md` +- `docs/project-structure.md` +- `docs/runtime-flows.md` +- `jarca_ledger_mcp/adapters/mcp/server.py` +- `jarca_ledger_mcp/application/imports_service.py` +- `tests/test_import_provenance_policy.py` + +## Additional Execution Evidence (Slice 02B) +- `. .venv/bin/activate && python -m ruff check jarca_ledger_mcp/application/imports_service.py jarca_ledger_mcp/adapters/mcp/server.py tests/test_import_provenance_policy.py` + - result: pass (`All checks passed!`) +- `. .venv/bin/activate && python -m pytest -q tests/test_import_provenance_policy.py` + - result: pass (`11 passed`) +- `. .venv/bin/activate && python -m ruff check .` + - result: pass (`All checks passed!`) +- `. .venv/bin/activate && python -m pytest -q` + - result: pass (`59 passed, 4 skipped`) + +## Updated Residual Risks / Follow-up +- Independent QC, PR creation/push, GitHub CI, and post-merge `confirm-req-implemented` remain pending. +- StagePilot doctor entrypoint is not present in the current workspace/runtime; this should be classified during QC as environment/tooling debt unless the lead requires a separate setup action before closure. diff --git a/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/verification.md b/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/verification.md index 2fb7529..1eca6b8 100644 --- a/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/verification.md +++ b/docs/batches/bat-017_20260708_import-provenance-sensitive-data-policy/verification.md @@ -1,11 +1,20 @@ # BAT-017 Verification Evidence -- Status: not-started +- Status: ready-for-qc - Created At (KST): 2026-07-08 11:33 KST ## Verification Commands -Pending. +- `. .venv/bin/activate && python -m ruff check jarca_ledger_mcp/application/imports_service.py jarca_ledger_mcp/adapters/mcp/server.py tests/test_import_provenance_policy.py` + - result: pass (`All checks passed!`) +- `. .venv/bin/activate && python -m pytest -q tests/test_import_provenance_policy.py` + - result: pass (`11 passed`) +- `. .venv/bin/activate && python -m ruff check .` + - result: pass (`All checks passed!`) +- `. .venv/bin/activate && python -m pytest -q` + - result: pass (`59 passed, 4 skipped`) +- StagePilot doctor + - result: blocked in current workspace/runtime because no discoverable `stagepilot-doctor` entrypoint or package was present; needs QC classification as pre-existing environment/tooling debt vs. closure blocker. ## Expected Gates @@ -16,3 +25,9 @@ Pending. - fixture/log sensitive-field scan - affected dashboard smoke, if applicable - StagePilot doctor with pre-existing debt separated from BAT-017 findings + +## Batch-specific verification notes +- Full pytest includes the affected import/privacy regression suite and leaves 4 skips in place without introducing new failures. +- Import batch debug-gated exposure now requires explicit `debug_provenance_gate=True`; the regression suite covers both deny-without-gate and allow-with-gate behavior. +- BAT-017 touched MCP import batch exposure behavior but did not modify dashboard code paths directly; no additional dashboard-only code change was required in this slice. +- QC still needs to independently inspect the changed surfaces, validate the documentation updates, and classify the missing StagePilot doctor entrypoint. diff --git a/docs/data-model.md b/docs/data-model.md index 8334a2e..2eaa018 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -4,8 +4,8 @@ - Baseline Seed: `.stage-pilot/bootstrap/baseline.yaml` - Status: draft - Owner: Jarca / project maintainer -- Last Updated (KST): 2026-07-01 -- Source Discovery / Batch: bootstrap-baseline, BAT-008, BAT-009 +- Last Updated (KST): 2026-07-09 +- Source Discovery / Batch: bootstrap-baseline, BAT-008, BAT-009, REQ-021/BAT-017 ## Project Summary - Jarca 가계부 전용 MCP 서버와 Streamlit 대시보드로 계좌/카드 거래, 예산, 분류 규칙, import, 현금흐름 분석을 SQLite에 저장하고 운영한다. @@ -107,6 +107,7 @@ - `category` / `subcategory` :: text :: household classification - `merchant`, `counterparty`, `description`, `payment_method` :: text :: evidence and display fields - `source`, `external_id`, `import_batch_id`, `raw_summary`, `raw_payload` :: text :: provenance and audit fields + - `raw_payload` write policy is `minimize`: only source-specific allowlisted normalized keys may be persisted; raw document dumps, full file paths, passwords, tokens/secrets, and arbitrary parser columns are excluded. - `memo` :: text :: operator notes and reconciliation tags - `status` :: text :: active/deleted/review lifecycle state where used - Relationships: @@ -160,13 +161,15 @@ - Key Fields: - `id` :: text :: import batch identifier - `source` :: text :: source kind - - `file_path` / source metadata where stored + - `filename` :: text :: masked basename plus source-hash prefix for operator correlation; full local source paths must not be stored on new writes + - `memo` :: text/json :: compact batch metadata such as source hash, validation summary, or source-specific sheet/page references used for read-time provenance shaping - `inserted_count`, `skipped_duplicate_count`, `created_at`, `rolled_back_at` - Relationships: - Transaction rows reference `import_batch_id`. - State Rules: - Rollback should soft-delete by default. - Duplicate imports with `skip_duplicates=True` may insert zero rows; that is not automatically an error. + - Read surfaces should expose sanitized provenance envelopes: default `minimal`, explicit operator-facing `operator-summary`, and explicit debug-gated diagnostics without full document/path leakage. ## Entity: Budget - Source of Truth: budget table/service functions @@ -224,7 +227,7 @@ - Operational backup/status evidence may include only path, size, mtime, checksum, integrity, row counts, and aggregate totals. - Validation DB copies should live outside the repository and should be selected explicitly via `JARCA_LEDGER_DB_PATH` when used. - Passwords for encrypted imports are runtime-only inputs and are not part of the persisted model. -- Raw payloads may contain sensitive financial evidence and should not be committed to git. +- Raw payloads may contain sensitive financial evidence and should not be committed to git; BAT-017 constrains new writes to minimized allowlisted payloads only and leaves legacy row cleanup to a separate approved follow-up. - ReviewDecision `target_payload` is sanitized to omit `raw_payload`, passwords, tokens, secrets, and file paths before persistence. ## Current Gaps / Planned Changes @@ -234,6 +237,7 @@ - BAT-009 implements manual asset CRUD mutation and month-end snapshot preview/close/correction service/MCP/dashboard workflow. Future work may add automatic market/real-estate valuation or scheduled month-end close. ## Change Log +- 2026-07-09 KST: REQ-021/BAT-017 기준으로 import provenance exposure policy, `transactions.raw_payload=minimize`, `import_batches.filename` masked-basename persistence, and explicit debug gate expectations를 반영. - 2026-06-26 22:24 KST: Created by `bootstrap-baseline` from repository observation. - 2026-06-27 10:17 KST: BAT-002/REQ-002 기준으로 LedgerBackupArtifact와 metadata-only evidence 정책 추가. - 2026-06-29 KST: REQ-003/BAT-003 기준으로 `accounts.statement_status`와 수동 잔액대조 preview 모델 추가. diff --git a/docs/project-structure.md b/docs/project-structure.md index 76ee531..f1635db 100644 --- a/docs/project-structure.md +++ b/docs/project-structure.md @@ -4,8 +4,8 @@ - Baseline Seed: `.stage-pilot/bootstrap/baseline.yaml` - Status: draft - Owner: Jarca / project maintainer -- Last Updated (KST): 2026-07-08 -- Source Discovery / Batch: bootstrap-baseline, BAT-006, BAT-007, BAT-010, BAT-012, BAT-013, BAT-014, dcy-013/BAT-015, dcy-012/REQ-020/BAT-016 +- Last Updated (KST): 2026-07-09 +- Source Discovery / Batch: bootstrap-baseline, BAT-006, BAT-007, BAT-010, BAT-012, BAT-013, BAT-014, dcy-013/BAT-015, dcy-012/REQ-020/BAT-016, REQ-021/BAT-017 ## Project Summary - Jarca 가계부 전용 MCP 서버와 Streamlit 대시보드로 계좌/카드 거래, 예산, 분류 규칙, import, 자산/스냅샷, 현금흐름 분석을 SQLite에 저장하고 운영한다. @@ -221,7 +221,8 @@ jarca-ledger-mcp/ ## Shared Boundaries - `cashflow_kind` is the boundary between actual household income/expense and account movement/exclusion semantics. - `liquidity_bucket` is the account/manual-asset boundary for liquidity/composition/debt/net-worth views. -- Import evidence fields (`source`, `external_id`, `raw_summary`, `raw_payload`, `counterparty`, `import_batch_id`) are audit boundaries and should be preserved. +- Import evidence fields (`source`, `external_id`, `raw_summary`, `raw_payload`, `counterparty`, `import_batch_id`) are audit boundaries and should be preserved, but BAT-017 constrains new persistence/response behavior to normalized provenance plus source-specific minimized payload allowlists. +- `import_batches.filename` is an operator-correlation field, not a raw path cache: new writes must use masked basename plus source-hash prefix, and read surfaces must expose provenance at `minimal` by default unless an explicit operator/debug path is invoked. - Manual reconciliation is represented through memo tags and narrow transaction updates, not broad rewrites. - Browser evidence artifacts are generated from synthetic/validation DBs and should not embed real financial identifiers or raw source payloads. - CI gate fixtures are generated at runtime under temp space by `scripts/ci_fixture_db.py`; CI/AppTest/browser smoke must not read the default operating DB path. @@ -237,6 +238,7 @@ jarca-ledger-mcp/ - Update this document when a batch changes the approved structure baseline. ## Change Log +- 2026-07-09 KST: REQ-021/BAT-017 기준으로 import provenance sanitizer 경계, masked filename persistence, explicit debug gate wiring, and baseline doc ownership을 반영. - 2026-06-26 22:24 KST: Created by `bootstrap-baseline` from repository observation. - 2026-06-29 KST: REQ-004/BAT-004 기준으로 `scripts/streamlit_browser_evidence.py` 및 browser evidence artifact 구조 추가. - 2026-06-29 KST: REQ-005/BAT-005 기준으로 dashboard operations hub와 service asset liquidity aggregation 책임 추가. diff --git a/docs/runtime-flows.md b/docs/runtime-flows.md index 31162f9..f7804d8 100644 --- a/docs/runtime-flows.md +++ b/docs/runtime-flows.md @@ -4,8 +4,8 @@ - Baseline Seed: `.stage-pilot/bootstrap/baseline.yaml` - Status: draft - Owner: Jarca / project maintainer -- Last Updated (KST): 2026-07-08 -- Source Discovery / Batch: bootstrap-baseline, BAT-006, BAT-007, BAT-008, BAT-010, BAT-012, BAT-013, BAT-014, dcy-013/BAT-015, dcy-012/REQ-020/BAT-016 +- Last Updated (KST): 2026-07-09 +- Source Discovery / Batch: bootstrap-baseline, BAT-006, BAT-007, BAT-008, BAT-010, BAT-012, BAT-013, BAT-014, dcy-013/BAT-015, dcy-012/REQ-020/BAT-016, REQ-021/BAT-017 - Primary Runtime: mixed ## Project Summary @@ -55,9 +55,14 @@ 2. MCP adapter forwards arguments to the matching callable exposed by `adapters/mcp/tool_contracts.py`. 3. Service facade routes into the relevant application module. 4. Application code opens SQLite through `infrastructure/sqlite` helpers, validates inputs, and performs the operation. -5. Application returns a dictionary payload with success data or error information. +5. Application returns a dictionary payload with success data or error information; import-related provenance is expected to be sanitized for the requested exposure level. 6. Hermes surfaces the tool result to the user or uses it in a follow-up workflow. +### MCP provenance exposure boundary +- Default finance-agent/tool responses use `minimal` provenance exposure. +- Operator-facing surfaces may request `operator-summary` when masked filename, source-hash prefix, or validation summary are needed for audit triage. +- Debug details require an explicit debug-gated path and still exclude passwords, tokens/secrets, and full source document dumps by default. + ## Flow: `Streamlit dashboard operation` 1. Operator starts `streamlit run dashboard/app.py --server.address 0.0.0.0 --server.port 8501`. 2. Dashboard reads the configured/default SQLite database. @@ -84,8 +89,8 @@ ## Flow: `Import preview and import` 1. Operator chooses a supported source file: KB Bank PDF, Toss Bank PDF, Toss Securities PDF, SeoulPay XLSX, or CSV. 2. Preview function routes through `imports_service.py`, which calls the parser seam under `infrastructure/parsers/*` and reports counts, sample rows, duplicate status, and classification preview. -3. On explicit import, `imports_service.py` creates/fetches the target account, inserts non-duplicate transactions, writes provenance, and records an import batch. -4. Reports and dashboard views read the inserted rows using shared category/cashflow semantics. +3. On explicit import, `imports_service.py` creates/fetches the target account, inserts non-duplicate transactions, writes minimized/raw-payload-allowlisted provenance, and records an import batch with masked filename plus source-hash prefix instead of a full local path. +4. Reports, MCP tools, and dashboard views read the inserted rows using shared category/cashflow semantics while returning normalized provenance metadata rather than raw document evidence by default. 5. If an import is wrong, rollback uses import batch data and soft-deletes by default. ## Flow: `Categorization rule lifecycle` @@ -155,6 +160,7 @@ - Update this document when a batch changes the approved flow baseline. ## Change Log +- 2026-07-09 KST: REQ-021/BAT-017 기준으로 import provenance exposure flow, operator-summary handling, and explicit debug gate requirement를 반영. - 2026-06-26 22:24 KST: Created by `bootstrap-baseline` from repository observation. - 2026-06-27 10:17 KST: BAT-002/REQ-002 기준으로 운영 backup/status/list/hash 및 restore 제한 flow 추가. - 2026-06-29 KST: REQ-003/BAT-003 기준으로 statement-unavailable 수동 잔액대조 flow 추가. diff --git a/jarca_ledger_mcp/adapters/mcp/server.py b/jarca_ledger_mcp/adapters/mcp/server.py index 0eb342c..18a665a 100644 --- a/jarca_ledger_mcp/adapters/mcp/server.py +++ b/jarca_ledger_mcp/adapters/mcp/server.py @@ -312,9 +312,21 @@ def import_toss_bank_pdf(file_path: str, account_id: str = "", skip_duplicates: return contracts.import_toss_bank_pdf(file_path, account_id, skip_duplicates) @mcp.tool() -def list_ledger_import_batches(source: str = "", include_rolled_back: bool = True, limit: int = 50) -> dict: +def list_ledger_import_batches( + source: str = "", + include_rolled_back: bool = True, + limit: int = 50, + provenance_exposure: str = "minimal", + debug_provenance_gate: bool = False, +) -> dict: """List CSV/PDF/API import batches.""" - return contracts.list_ledger_import_batches(source, include_rolled_back, limit) + return contracts.list_ledger_import_batches( + source, + include_rolled_back, + limit, + provenance_exposure=provenance_exposure, + debug_provenance_gate=debug_provenance_gate, + ) @mcp.tool() def rollback_ledger_import_batch(import_batch_id: str, hard_delete: bool = False) -> dict: diff --git a/jarca_ledger_mcp/application/imports_service.py b/jarca_ledger_mcp/application/imports_service.py index f8315a2..ef652b6 100644 --- a/jarca_ledger_mcp/application/imports_service.py +++ b/jarca_ledger_mcp/application/imports_service.py @@ -1,5 +1,13 @@ from __future__ import annotations +from ..provenance_policy import ( + EXPOSURE_LEVEL_MINIMAL, + file_sha256, + redacted_source_file_label, + sanitize_csv_preview_rows, + sanitize_provenance_for_exposure, +) + from .shared import ( Any, Path, @@ -29,10 +37,9 @@ def preview_ledger_csv_import(file_path: str, source: str = "generic", db_path: return fail("not_found", f"CSV file not found: {path}") with path.open("r", encoding="utf-8-sig", newline="") as f: rows=list(csv.DictReader(f)) - preview=[] - for r in rows[:20]: - preview.append(r) - return ok(source=source, filename=str(path), row_count=len(rows), preview=preview, required_columns=["transaction_date/date", "amount", "type optional", "category optional", "merchant/description optional"]) + source_hash = file_sha256(path) + preview = sanitize_csv_preview_rows(rows, limit=20) + return ok(source=source, filename=redacted_source_file_label(str(path), source_hash), source_hash=source_hash, row_count=len(rows), preview=preview, required_columns=["transaction_date/date", "amount", "type optional", "category optional", "merchant/description optional"]) def preview_kb_bank_pdf_import(file_path: str, password: str, preview_limit: int = 20, db_path: str | None = None) -> dict[str, Any]: @@ -56,7 +63,7 @@ def preview_kb_bank_pdf_import(file_path: str, password: str, preview_limit: int preview = [transaction_preview(tx, source_hash) for tx in transactions[: max(1, min(int(preview_limit), 100))]] return ok( source="kb_bank_pdf", - file_path=parsed["file_path"], + file_path=redacted_source_file_label(parsed["file_path"], source_hash), source_hash=source_hash, page_count=parsed["page_count"], metadata=parsed["metadata"], @@ -137,7 +144,7 @@ def import_kb_bank_pdf(file_path: str, password: str, account_id: str = "", skip ( batch_id, "kb_bank_pdf", - parsed["file_path"], + redacted_source_file_label(parsed["file_path"], source_hash), ts, 0, "active", @@ -387,7 +394,7 @@ def preview_seoulpay_xlsx_import(file_path: str, password: str = "", preview_lim preview = [seoulpay_transaction_preview(tx, source_hash) for tx in transactions[: max(1, min(int(preview_limit), 100))]] return ok( source="seoulpay_xlsx", - file_path=parsed["file_path"], + file_path=redacted_source_file_label(parsed["file_path"], source_hash), source_hash=source_hash, sheet_name=parsed["sheet_name"], metadata=parsed["metadata"], @@ -431,7 +438,7 @@ def import_seoulpay_xlsx(file_path: str, password: str = "", account_id: str = " ( batch_id, "seoulpay_xlsx", - parsed["file_path"], + redacted_source_file_label(parsed["file_path"], source_hash), ts, 0, "active", @@ -553,7 +560,7 @@ def preview_toss_bank_pdf_import(file_path: str, preview_limit: int = 20, db_pat item["amount"] += int(tx.amount) return ok( source="toss_bank_pdf", - file_path=parsed["file_path"], + file_path=redacted_source_file_label(parsed["file_path"], source_hash), source_hash=source_hash, page_count=parsed["page_count"], metadata=parsed["metadata"], @@ -597,7 +604,7 @@ def import_toss_bank_pdf(file_path: str, account_id: str = "", skip_duplicates: ( batch_id, "toss_bank_pdf", - parsed["file_path"], + redacted_source_file_label(parsed["file_path"], source_hash), ts, 0, "active", @@ -786,7 +793,7 @@ def preview_toss_securities_pdf_import(file_path: str, preview_limit: int = 20, item["amount"] += int(tx.amount) return ok( source="toss_securities_pdf", - file_path=parsed["file_path"], + file_path=redacted_source_file_label(parsed["file_path"], source_hash), source_hash=source_hash, page_count=parsed["page_count"], metadata=parsed["metadata"], @@ -827,7 +834,7 @@ def import_toss_securities_pdf(file_path: str, account_id: str = "", skip_duplic ( batch_id, "toss_securities_pdf", - parsed["file_path"], + redacted_source_file_label(parsed["file_path"], source_hash), ts, 0, "active", @@ -916,7 +923,64 @@ def import_toss_securities_pdf(file_path: str, account_id: str = "", skip_duplic ) -def list_ledger_import_batches(source: str = "", include_rolled_back: bool = True, limit: int = 50, db_path: str | None = None) -> dict[str, Any]: +def list_ledger_import_batches( + source: str = "", + include_rolled_back: bool = True, + limit: int = 50, + provenance_exposure: str = EXPOSURE_LEVEL_MINIMAL, + debug_provenance_gate: bool = False, + db_path: str | None = None, +) -> dict[str, Any]: + return list_ledger_import_batches_with_provenance( + source=source, + include_rolled_back=include_rolled_back, + limit=limit, + provenance_exposure=provenance_exposure, + debug_provenance_gate=debug_provenance_gate, + db_path=db_path, + ) + + +def _load_batch_memo(memo: Any) -> dict[str, Any]: + if isinstance(memo, dict): + return memo + if not isinstance(memo, str) or not memo.strip(): + return {} + try: + parsed = json.loads(memo) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _batch_provenance(row: dict[str, Any]) -> dict[str, Any]: + memo = _load_batch_memo(row.get("memo")) + provenance = { + "source": row.get("source") or "", + "parser": row.get("source") or "", + "filename": row.get("filename") or "", + "import_batch_id": row.get("id") or "", + "source_hash": memo.get("source_hash") or "", + "validation": memo.get("validation") if isinstance(memo.get("validation"), dict) else {}, + } + if memo.get("sheet_name"): + provenance["sheet_name"] = memo["sheet_name"] + return provenance + + +def list_ledger_import_batches_with_provenance( + source: str = "", + include_rolled_back: bool = True, + limit: int = 50, + provenance_exposure: str = EXPOSURE_LEVEL_MINIMAL, + debug_provenance_gate: bool = False, + db_path: str | None = None, +) -> dict[str, Any]: + if provenance_exposure == "debug-gated" and not debug_provenance_gate: + return fail( + "forbidden", + "debug-gated provenance requires explicit debug_provenance_gate=True", + ) clauses=[]; params=[] if source: clauses.append("source=?"); params.append(source) @@ -926,7 +990,20 @@ def list_ledger_import_batches(source: str = "", include_rolled_back: bool = Tru params.append(max(1, min(int(limit), 500))) with db(db_path) as conn: rows=conn.execute(f"SELECT * FROM import_batches{where} ORDER BY imported_at DESC LIMIT ?", params).fetchall() - return ok(import_batches=[row_to_dict(r) for r in rows], count=len(rows)) + try: + items = [] + for row in rows: + item = row_to_dict(row) + provenance = _batch_provenance(item) + item["memo"] = "" + item["provenance"] = sanitize_provenance_for_exposure( + provenance, + provenance_exposure, + ) + items.append(item) + except ValueError as exc: + return fail("validation_error", str(exc)) + return ok(import_batches=items, count=len(items)) def rollback_ledger_import_batch(import_batch_id: str, hard_delete: bool = False, db_path: str | None = None) -> dict[str, Any]: diff --git a/jarca_ledger_mcp/kb_bank_pdf.py b/jarca_ledger_mcp/kb_bank_pdf.py index 27fab2d..5ac3899 100644 --- a/jarca_ledger_mcp/kb_bank_pdf.py +++ b/jarca_ledger_mcp/kb_bank_pdf.py @@ -1,13 +1,14 @@ from __future__ import annotations import hashlib -import json import re from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Any +from .provenance_policy import allowlisted_json_payload + try: import fitz # PyMuPDF except ImportError: # pragma: no cover - exercised as runtime config error @@ -55,7 +56,7 @@ class KbPdfTransaction: return hashlib.sha256(raw.encode("utf-8")).hexdigest() def raw_payload(self) -> str: - return json.dumps( + return allowlisted_json_payload( { "page": self.page, "transaction_datetime": self.transaction_datetime, @@ -67,8 +68,7 @@ class KbPdfTransaction: "remittance_memo": self.remittance_memo, "branch": self.branch, }, - ensure_ascii=False, - separators=(",", ":"), + allowed_keys=("page",), ) diff --git a/jarca_ledger_mcp/provenance_policy.py b/jarca_ledger_mcp/provenance_policy.py new file mode 100644 index 0000000..0e091a2 --- /dev/null +++ b/jarca_ledger_mcp/provenance_policy.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path +from typing import Any, Iterable, Mapping + +SOURCE_HASH_PREFIX_LENGTH = 12 +CSV_PREVIEW_ALLOWED_FIELDS = ( + "transaction_date", + "date", + "amount", + "type", + "category", + "subcategory", + "merchant", + "description", +) + +EXPOSURE_LEVEL_MINIMAL = "minimal" +EXPOSURE_LEVEL_OPERATOR_SUMMARY = "operator-summary" +EXPOSURE_LEVEL_DEBUG_GATED = "debug-gated" +EXPOSURE_LEVELS = ( + EXPOSURE_LEVEL_MINIMAL, + EXPOSURE_LEVEL_OPERATOR_SUMMARY, + EXPOSURE_LEVEL_DEBUG_GATED, +) + +_FORBIDDEN_KEY_HINTS = ( + "password", + "pwd", + "token", + "secret", + "credential", + "raw_document", + "raw_text", + "raw_dump", + "raw_payload_dump", +) +_PATH_KEY_HINTS = ("path", "filename", "filepath", "file_path", "source_path", "local_path") +_PATH_PREFIX_RE = re.compile(r"^[A-Za-z]:[\\/]|^(?:/|\\)") + +_TOPLEVEL_FIELDS = { + EXPOSURE_LEVEL_MINIMAL: { + "source", + "source_hash", + "source_hash_prefix", + "parser", + "import_batch_id", + "external_id", + "dedupe_key", + }, + EXPOSURE_LEVEL_OPERATOR_SUMMARY: { + "source", + "source_hash", + "source_hash_prefix", + "parser", + "filename", + "import_batch_id", + "external_id", + "dedupe_key", + "page", + "sheet", + "row_number", + "sheet_name", + "validation", + }, + EXPOSURE_LEVEL_DEBUG_GATED: { + "source", + "source_hash", + "source_hash_prefix", + "parser", + "filename", + "import_batch_id", + "external_id", + "dedupe_key", + "page", + "sheet", + "row_number", + "sheet_name", + "validation", + "raw_payload", + "debug", + "diagnostics", + }, +} + +# Default field allowlists intentionally stay intentionally narrow and source-agnostic. +_RAW_PAYLOAD_ALLOWLIST = { + EXPOSURE_LEVEL_MINIMAL: frozenset(), + EXPOSURE_LEVEL_OPERATOR_SUMMARY: frozenset( + { + "row_index", + "page", + "sheet", + "transaction_date", + "transaction_datetime", + "type", + "amount", + "code", + "section", + "action", + "name", + "counterparty", + "merchant", + } + ), + EXPOSURE_LEVEL_DEBUG_GATED: frozenset( + { + "row_index", + "page", + "sheet", + "transaction_date", + "transaction_datetime", + "type", + "amount", + "code", + "section", + "action", + "name", + "counterparty", + "merchant", + "transaction_no", + "business_no", + "merchant_id", + "channel", + "balance_after", + "quantity", + "unit_price", + "fee", + "tax", + "row_number", + "row_ref", + } + ), +} + + +def source_hash_prefix(source_hash: str, length: int = SOURCE_HASH_PREFIX_LENGTH) -> str: + return (source_hash or "").strip()[:length] + + +def redacted_source_file_label(file_path: str, source_hash: str = "") -> str: + basename = Path(str(file_path or "")).name or "source" + token = source_hash_prefix(source_hash) + return f"{basename}#{token}" if token else basename + + +def file_sha256(file_path: str | Path) -> str: + return hashlib.sha256(Path(file_path).read_bytes()).hexdigest() + + +def allowlisted_json_payload(payload: Mapping[str, Any], allowed_keys: Iterable[str]) -> str: + cleaned = { + key: payload[key] + for key in allowed_keys + if key in payload and payload[key] not in (None, "", [], {}) + } + return json.dumps(cleaned, ensure_ascii=False, separators=(",", ":")) + + +def sanitize_csv_preview_rows(rows: list[Mapping[str, Any]], limit: int = 20) -> list[dict[str, Any]]: + preview: list[dict[str, Any]] = [] + for row_number, row in enumerate(rows[: max(0, limit)], start=1): + item = {"row_number": row_number} + for key in CSV_PREVIEW_ALLOWED_FIELDS: + value = row.get(key) + if value not in (None, ""): + item[key] = value + preview.append(item) + return preview + + +def _is_forbidden_key(key: str) -> bool: + lower = key.lower() + return any(token in lower for token in _FORBIDDEN_KEY_HINTS) + + +def _is_path_key(key: str) -> bool: + lower = key.lower() + return any(lower.endswith(path_key) or lower == path_key for path_key in _PATH_KEY_HINTS) + + +def _is_full_path(value: str) -> bool: + if not value: + return False + return bool(_PATH_PREFIX_RE.search(value)) and ("/" in value or "\\" in value) + + +def _sanitize_mapping( + data: Mapping[str, Any], + *, + allowed_keys: set[str] | None, + source_hash: str, +) -> dict[str, Any]: + sanitized: dict[str, Any] = {} + for key, value in data.items(): + key_name = str(key) + if allowed_keys is not None and key_name not in allowed_keys: + continue + if _is_forbidden_key(key_name): + continue + if _is_path_key(key_name) and isinstance(value, str) and _is_full_path(value): + sanitized[key_name] = redacted_source_file_label(value, source_hash) + continue + if isinstance(value, Mapping): + nested = _sanitize_mapping(value, allowed_keys=allowed_keys, source_hash=source_hash) + if nested: + sanitized[key_name] = nested + continue + if isinstance(value, list): + if value == []: + sanitized[key_name] = [] + continue + + nested_list: list[Any] = [] + for item in value: + if isinstance(item, Mapping): + nested_item = _sanitize_mapping(item, allowed_keys=allowed_keys, source_hash=source_hash) + if nested_item: + nested_list.append(nested_item) + elif isinstance(item, str): + if _is_forbidden_key(item) or _is_full_path(item): + continue + nested_list.append(item) + else: + nested_list.append(item) + if nested_list: + sanitized[key_name] = nested_list + continue + if isinstance(value, str): + if _is_forbidden_key(value) or _is_full_path(value): + continue + if value is not None: + sanitized[key_name] = value + return sanitized + + +def _normalize_raw_payload(raw_payload: Any, exposure_level: str, source_hash: str) -> dict[str, Any]: + if isinstance(raw_payload, str): + try: + parsed_payload = json.loads(raw_payload) + except Exception: + return {} + else: + parsed_payload = raw_payload + + if not isinstance(parsed_payload, Mapping): + return {} + + allowlist = set(_RAW_PAYLOAD_ALLOWLIST[exposure_level]) + if not allowlist: + return {} + + return _sanitize_mapping(parsed_payload, allowed_keys=allowlist, source_hash=source_hash) + + +def sanitize_provenance( + provenance: Mapping[str, Any], + exposure_level: str = EXPOSURE_LEVEL_MINIMAL, +) -> dict[str, Any]: + if exposure_level not in EXPOSURE_LEVELS: + raise ValueError(f"unsupported exposure level: {exposure_level}") + + source_hash = str(provenance.get("source_hash", "")) + source_hash_prefix_value = source_hash_prefix(source_hash) + allowed_fields = _TOPLEVEL_FIELDS[exposure_level] + + sanitized: dict[str, Any] = {"source_hash_prefix": source_hash_prefix_value} + + for key in sorted(allowed_fields): + if key == "source_hash_prefix": + sanitized[key] = source_hash_prefix_value + continue + if key not in provenance: + continue + if key in {"raw_payload", "debug", "diagnostics"} and key in provenance: + if key in {"raw_payload"}: + value = _normalize_raw_payload( + provenance[key], + exposure_level=exposure_level, + source_hash=source_hash, + ) + else: + value = _sanitize_mapping(provenance[key], allowed_keys=None, source_hash=source_hash) if isinstance(provenance[key], Mapping) else provenance[key] + if value: + sanitized[key] = value + continue + + value = provenance[key] + if _is_forbidden_key(str(key)): + continue + if isinstance(value, Mapping): + value = _sanitize_mapping(value, allowed_keys=None, source_hash=source_hash) + if not value: + continue + sanitized[key] = value + continue + + if isinstance(value, list): + nested_list: list[Any] = [] + for item in value: + if isinstance(item, Mapping): + item_value = _sanitize_mapping(item, allowed_keys=None, source_hash=source_hash) + if item_value: + nested_list.append(item_value) + elif isinstance(item, str): + if _is_forbidden_key(item) or _is_full_path(item): + continue + nested_list.append(item) + else: + nested_list.append(item) + if nested_list: + sanitized[key] = nested_list + continue + + if _is_path_key(str(key)) and isinstance(value, str) and _is_full_path(value): + sanitized[key] = redacted_source_file_label(value, source_hash) + continue + + if isinstance(value, str): + if _is_forbidden_key(value) or _is_full_path(value): + continue + + if value is not None: + sanitized[key] = value + + if "parser" not in sanitized and "source" in provenance: + sanitized["parser"] = provenance["source"] + + if exposure_level != EXPOSURE_LEVEL_MINIMAL: + for path_key in ("file_path", "source_path", "local_path", "filename"): + raw_path = provenance.get(path_key) + if isinstance(raw_path, str) and _is_full_path(raw_path): + sanitized["filename"] = redacted_source_file_label(raw_path, source_hash) + break + + # Keep raw document/text dumps suppressed even on debug output. + for forbidden_key in ("raw_document", "raw_text", "raw_dump", "raw_payload_dump"): + sanitized.pop(forbidden_key, None) + + return sanitized + + +def sanitize_provenance_envelope( + provenance: Mapping[str, Any], + exposure_level: str = EXPOSURE_LEVEL_MINIMAL, +) -> dict[str, Any]: + return sanitize_provenance(provenance, exposure_level=exposure_level) + + +def sanitize_provenance_for_exposure( + provenance: Mapping[str, Any], + exposure_level: str = EXPOSURE_LEVEL_MINIMAL, +) -> dict[str, Any]: + return sanitize_provenance(provenance, exposure_level=exposure_level) diff --git a/jarca_ledger_mcp/seoulpay_xlsx.py b/jarca_ledger_mcp/seoulpay_xlsx.py index e6d6806..b1c4b55 100644 --- a/jarca_ledger_mcp/seoulpay_xlsx.py +++ b/jarca_ledger_mcp/seoulpay_xlsx.py @@ -7,6 +7,8 @@ from datetime import datetime from pathlib import Path from typing import Any +from .provenance_policy import allowlisted_json_payload + @dataclass(frozen=True) class SeoulPayTransaction: @@ -38,9 +40,7 @@ class SeoulPayTransaction: return f"seoulpay_xlsx:{source_hash}:{base}" def raw_payload(self) -> str: - import json - - return json.dumps( + return allowlisted_json_payload( { "row_index": self.row_index, "channel": self.channel, @@ -57,7 +57,7 @@ class SeoulPayTransaction: "vat": self.vat, "tax_status": self.tax_status, }, - ensure_ascii=False, + allowed_keys=("row_index",), ) diff --git a/jarca_ledger_mcp/toss_bank_pdf.py b/jarca_ledger_mcp/toss_bank_pdf.py index f178a37..48864c5 100644 --- a/jarca_ledger_mcp/toss_bank_pdf.py +++ b/jarca_ledger_mcp/toss_bank_pdf.py @@ -1,7 +1,6 @@ from __future__ import annotations import hashlib -import json import re from dataclasses import dataclass from pathlib import Path @@ -9,6 +8,8 @@ from typing import Any import fitz +from .provenance_policy import allowlisted_json_payload + DATETIME_RE = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$") AMOUNT_RE = re.compile(r"^[+-]?\d{1,3}(?:,\d{3})*$") @@ -32,7 +33,7 @@ class TossBankTransaction: return f"toss_bank_pdf:{source_hash}:{self.row_index}:{self.transaction_datetime}:{self.amount}" def raw_payload(self) -> str: - return json.dumps( + return allowlisted_json_payload( { "row_index": self.row_index, "transaction_datetime": self.transaction_datetime, @@ -44,7 +45,7 @@ class TossBankTransaction: "raw_summary": self.raw_summary, "raw_lines": self.raw_lines, }, - ensure_ascii=False, + allowed_keys=("row_index",), ) diff --git a/jarca_ledger_mcp/toss_securities_pdf.py b/jarca_ledger_mcp/toss_securities_pdf.py index 0339bca..a1774e5 100644 --- a/jarca_ledger_mcp/toss_securities_pdf.py +++ b/jarca_ledger_mcp/toss_securities_pdf.py @@ -9,6 +9,8 @@ from typing import Any import fitz +from .provenance_policy import allowlisted_json_payload + DATE_RE = re.compile(r"^\d{4}\.\d{2}\.\d{2}$") FOOTER_RE = re.compile(r"^\d+\s*/\s*\d+$") NUM_RE = re.compile(r"^-?\d{1,3}(?:,\d{3})*(?:\.\d+)?$|^-?\d+(?:\.\d+)?$") @@ -105,7 +107,7 @@ class TossSecuritiesTransaction: return "tosssec:" + hashlib.sha256(raw.encode("utf-8")).hexdigest() def raw_payload(self) -> str: - return json.dumps( + return allowlisted_json_payload( { "row_index": self.row_index, "section": self.section, @@ -123,7 +125,7 @@ class TossSecuritiesTransaction: "raw_numbers": self.raw_numbers, "raw_block": self.raw_block, }, - ensure_ascii=False, + allowed_keys=("row_index", "section", "quantity", "unit_price", "fee", "tax"), ) diff --git a/tests/test_import_provenance_policy.py b/tests/test_import_provenance_policy.py new file mode 100644 index 0000000..646fddb --- /dev/null +++ b/tests/test_import_provenance_policy.py @@ -0,0 +1,538 @@ +from __future__ import annotations + +import csv +import json +import sqlite3 +from pathlib import Path + +from jarca_ledger_mcp import service +from jarca_ledger_mcp.provenance_policy import ( + EXPOSURE_LEVEL_DEBUG_GATED, + EXPOSURE_LEVEL_MINIMAL, + EXPOSURE_LEVEL_OPERATOR_SUMMARY, + sanitize_provenance_for_exposure, + sanitize_provenance, + sanitize_provenance_envelope, +) +from jarca_ledger_mcp.kb_bank_pdf import KbPdfTransaction +from jarca_ledger_mcp.seoulpay_xlsx import SeoulPayTransaction +from jarca_ledger_mcp.toss_bank_pdf import TossBankTransaction +from jarca_ledger_mcp.toss_securities_pdf import TossSecuritiesTransaction + + +SECRET_SOURCE_PATH = "/very/secret/home/alice/Documents/private/statement.pdf" +SOURCE_HASH = "abcdef1234567890fedcba0987654321abcdef1234567890fedcba0987654321" + + +def _expected_filename(file_path: str, source_hash: str = SOURCE_HASH) -> str: + return f"{Path(file_path).name}#{source_hash[:12]}" + + +def _stored_batch_and_payloads(db_path: str) -> tuple[str, list[dict[str, object]]]: + with sqlite3.connect(db_path) as conn: + batch_filename = conn.execute("SELECT filename FROM import_batches ORDER BY imported_at DESC LIMIT 1").fetchone()[0] + payload_rows = conn.execute("SELECT raw_payload FROM transactions ORDER BY created_at").fetchall() + return batch_filename, [json.loads(row[0]) for row in payload_rows] + + +def test_preview_ledger_csv_import_redacts_full_path_and_raw_columns(tmp_path): + csv_path = tmp_path / "import-preview.csv" + with csv_path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=["date", "amount", "merchant", "secret_note", "local_path"]) + writer.writeheader() + writer.writerow( + { + "date": "2026-06-18", + "amount": "42000", + "merchant": "테스트상점", + "secret_note": "do not leak this raw note", + "local_path": "/Users/alice/Desktop/private.csv", + } + ) + + result = service.preview_ledger_csv_import(str(csv_path)) + + assert result["success"] is True + assert result["filename"] == _expected_filename(str(csv_path), result["source_hash"]) + assert result["preview"] == [{"row_number": 1, "date": "2026-06-18", "amount": "42000", "merchant": "테스트상점"}] + rendered = json.dumps(result, ensure_ascii=False) + assert str(tmp_path) not in rendered + assert "secret_note" not in rendered + assert "local_path" not in rendered + + +def test_preview_and_import_kb_bank_pdf_sanitize_path_and_payload(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": SECRET_SOURCE_PATH, + "source_hash": SOURCE_HASH, + "page_count": 4, + "metadata": {"account_name": "생활비", "account_number_masked": "****5704", "holder_name": "홍길동", "final_balance": 123456}, + "transactions": [ + KbPdfTransaction( + page=7, + transaction_datetime="2026-06-18T09:00:00", + transaction_date="2026-06-18", + raw_summary="적요-민감메모", + counterparty="테스트상대", + withdrawal=42000, + deposit=0, + balance_after=123456, + remittance_memo="숨기고싶은메모", + branch="강남지점", + ) + ], + "validation": {"ok": True, "errors": []}, + } + monkeypatch.setattr(service, "parse_kb_bank_pdf", lambda file_path, password: parsed) + + preview = service.preview_kb_bank_pdf_import("ignored.pdf", "pw", db_path=db_path) + imported = service.import_kb_bank_pdf("ignored.pdf", "pw", db_path=db_path) + + assert preview["success"] is True + assert preview["file_path"] == _expected_filename(SECRET_SOURCE_PATH) + assert SECRET_SOURCE_PATH not in json.dumps(preview, ensure_ascii=False) + assert imported["success"] is True + + batch_filename, payloads = _stored_batch_and_payloads(db_path) + assert batch_filename == _expected_filename(SECRET_SOURCE_PATH) + assert payloads == [{"page": 7}] + + +def test_preview_and_import_seoulpay_xlsx_sanitize_path_and_payload(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": "/very/secret/home/alice/Documents/private/seoulpay.xlsx", + "source_hash": SOURCE_HASH, + "sheet_name": "결제내역", + "metadata": {"holder_name": "홍길동", "period": "20260601~20260620"}, + "transactions": [ + SeoulPayTransaction( + row_index=11, + channel="서울페이", + transaction_no="TRX-001", + transaction_datetime="2026-06-18 18:27:46", + transaction_date="2026-06-18", + merchant="땡겨요", + business_no="2028102637", + merchant_id="AF2402815444", + transaction_kind="온라인 결제", + payment_kind="결제", + amount=42000, + supply_amount=38182, + service_fee=0, + vat=3818, + tax_status="과세", + ) + ], + "validation": {"ok": True, "parsed_rows": 1, "errors": []}, + } + monkeypatch.setattr(service, "parse_seoulpay_xlsx", lambda file_path, password="": parsed) + + preview = service.preview_seoulpay_xlsx_import("ignored.xlsx", db_path=db_path) + imported = service.import_seoulpay_xlsx("ignored.xlsx", db_path=db_path) + + assert preview["success"] is True + assert preview["file_path"] == _expected_filename(parsed["file_path"]) + assert parsed["file_path"] not in json.dumps(preview, ensure_ascii=False) + assert imported["success"] is True + + batch_filename, payloads = _stored_batch_and_payloads(db_path) + assert batch_filename == _expected_filename(parsed["file_path"]) + assert payloads == [{"row_index": 11}] + + +def test_preview_and_import_toss_bank_pdf_sanitize_path_and_payload(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": SECRET_SOURCE_PATH, + "source_hash": SOURCE_HASH, + "page_count": 3, + "metadata": {"holder_name": "홍길동", "account_name": "토스 통장", "account_number_masked": "100-1234-5678", "final_balance": 820000}, + "transactions": [ + TossBankTransaction( + row_index=9, + transaction_datetime="2026-06-18 09:00:00", + transaction_date="2026-06-18", + action="이체", + counterparty="테스트상대", + amount=12000, + type="expense", + balance_after=820000, + raw_summary="상대방 요약", + raw_lines=[SECRET_SOURCE_PATH, "ssh private key"], + ) + ], + "validation": {"ok": True, "parsed_rows": 1, "errors": []}, + } + monkeypatch.setattr(service, "parse_toss_bank_pdf", lambda file_path: parsed) + + preview = service.preview_toss_bank_pdf_import("ignored.pdf", db_path=db_path) + imported = service.import_toss_bank_pdf("ignored.pdf", db_path=db_path) + + assert preview["success"] is True + assert preview["file_path"] == _expected_filename(SECRET_SOURCE_PATH) + rendered = json.dumps(preview, ensure_ascii=False) + assert SECRET_SOURCE_PATH not in rendered + assert "raw_lines" not in rendered + assert imported["success"] is True + + batch_filename, payloads = _stored_batch_and_payloads(db_path) + assert batch_filename == _expected_filename(SECRET_SOURCE_PATH) + assert payloads == [{"row_index": 9}] + + +def test_preview_and_import_toss_securities_pdf_sanitize_path_and_payload(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": SECRET_SOURCE_PATH, + "source_hash": SOURCE_HASH, + "page_count": 5, + "metadata": {"holder_name": "홍길동", "account_name": "토스증권 계좌", "account_number_masked": "123-01-048829"}, + "transactions": [ + TossSecuritiesTransaction( + row_index=5, + section="USD", + transaction_date="2026-06-18", + action="외화증권배당금입금", + name="ACME(A123)", + code="A123", + amount=4300, + type="income", + quantity=1.5, + unit_price=25.2, + fee=120, + tax=33, + balance_after=910000, + raw_numbers=["1.5", "25.2", "4300"], + raw_block=[SECRET_SOURCE_PATH, "api_token=secret"], + ) + ], + "validation": {"ok": True, "parsed_rows": 1, "errors": []}, + } + monkeypatch.setattr(service, "parse_toss_securities_pdf", lambda file_path: parsed) + + preview = service.preview_toss_securities_pdf_import("ignored.pdf", db_path=db_path) + imported = service.import_toss_securities_pdf("ignored.pdf", db_path=db_path) + + assert preview["success"] is True + assert preview["file_path"] == _expected_filename(SECRET_SOURCE_PATH) + rendered = json.dumps(preview, ensure_ascii=False) + assert SECRET_SOURCE_PATH not in rendered + assert "raw_block" not in rendered + assert imported["success"] is True + + batch_filename, payloads = _stored_batch_and_payloads(db_path) + assert batch_filename == _expected_filename(SECRET_SOURCE_PATH) + assert payloads == [{"row_index": 5, "section": "USD", "quantity": 1.5, "unit_price": 25.2, "fee": 120, "tax": 33}] + + +def test_sanitize_provenance_by_exposure_levels_filters_sensitive_values(): + envelope = { + "source": "kb_bank_pdf", + "source_hash": SOURCE_HASH, + "file_path": SECRET_SOURCE_PATH, + "filename": SECRET_SOURCE_PATH, + "parser": "kb", + "import_batch_id": "batch-1", + "external_id": "ext-1", + "dedupe_key": "dedupe-1", + "row_number": 7, + "page": 3, + "sheet": "A1", + "validation": {"ok": True, "errors": []}, + "raw_payload": { + "row_index": 7, + "page": 3, + "action": "transfer", + "raw_text": "raw statement text", + "password": "p@ss", + "counterparty": "김대리", + "secret": "should-hide", + "transaction_path": "/very/sensitive/path.csv", + "raw_document": "full text dump", + }, + "debug": { + "parser_diag": "ok", + "raw_text": "should-be-hidden", + "raw_payload_dump": "hide-this", + "token": "token-001", + "message": "keep? no", + "notes": ["safe", "path=/tmp/should-not-leak", {"secret": "still-hidden", "message": "ok"}], + }, + "local_path": "/another/secret/path", + } + + minimal = sanitize_provenance(envelope, exposure_level=EXPOSURE_LEVEL_MINIMAL) + assert minimal["source"] == "kb_bank_pdf" + assert minimal["parser"] == "kb" + assert minimal["source_hash"] == SOURCE_HASH + assert minimal["source_hash_prefix"] == SOURCE_HASH[:12] + assert "file_path" not in minimal + assert "filename" not in minimal + assert "raw_payload" not in minimal + assert "debug" not in minimal + rendered = json.dumps(minimal, ensure_ascii=False) + assert SECRET_SOURCE_PATH not in rendered + assert "/tmp/should-not-leak" not in rendered + assert "password" not in rendered + assert "token" not in rendered + + operator_summary = sanitize_provenance( + envelope, + exposure_level=EXPOSURE_LEVEL_OPERATOR_SUMMARY, + ) + assert operator_summary["source"] == "kb_bank_pdf" + assert operator_summary["parser"] == "kb" + assert operator_summary["filename"] == _expected_filename(SECRET_SOURCE_PATH) + assert operator_summary["source_hash_prefix"] == SOURCE_HASH[:12] + assert operator_summary["validation"] == {"ok": True, "errors": []} + assert "raw_payload" not in operator_summary + assert operator_summary["filename"] == _expected_filename(SECRET_SOURCE_PATH) + assert "raw_text" not in json.dumps(operator_summary, ensure_ascii=False) + + debug = sanitize_provenance( + envelope, + exposure_level=EXPOSURE_LEVEL_DEBUG_GATED, + ) + assert debug["filename"] == _expected_filename(SECRET_SOURCE_PATH) + assert debug["source_hash_prefix"] == SOURCE_HASH[:12] + assert "raw_payload" in debug + debug_payload = debug["raw_payload"] + assert debug_payload == { + "row_index": 7, + "page": 3, + "action": "transfer", + "counterparty": "김대리", + } + assert "password" not in json.dumps(debug, ensure_ascii=False) + assert "raw_text" not in json.dumps(debug, ensure_ascii=False) + assert "token" not in json.dumps(debug, ensure_ascii=False) + assert "raw_document" not in json.dumps(debug, ensure_ascii=False) + + +def test_sanitize_provenance_envelope_alias_matches_core_sanitizer(): + envelope = { + "source": "toss_securities_pdf", + "source_hash": SOURCE_HASH, + "file_path": SECRET_SOURCE_PATH, + "raw_payload": '{"row_index":1,"page":2,"secret":"x"}', + } + + assert sanitize_provenance_envelope(envelope, EXPOSURE_LEVEL_MINIMAL) == sanitize_provenance( + envelope, + EXPOSURE_LEVEL_MINIMAL, + ) + assert sanitize_provenance_envelope(envelope, EXPOSURE_LEVEL_OPERATOR_SUMMARY) == sanitize_provenance( + envelope, + EXPOSURE_LEVEL_OPERATOR_SUMMARY, + ) + assert sanitize_provenance_envelope(envelope, EXPOSURE_LEVEL_DEBUG_GATED) == sanitize_provenance( + envelope, + EXPOSURE_LEVEL_DEBUG_GATED, + ) + + assert sanitize_provenance_for_exposure(envelope, EXPOSURE_LEVEL_OPERATOR_SUMMARY) == sanitize_provenance( + envelope, + EXPOSURE_LEVEL_OPERATOR_SUMMARY, + ) + + +def test_list_ledger_import_batches_defaults_to_minimal_provenance(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": SECRET_SOURCE_PATH, + "source_hash": SOURCE_HASH, + "page_count": 4, + "metadata": {"account_name": "생활비", "account_number_masked": "****5704", "holder_name": "홍길동", "final_balance": 123456}, + "transactions": [ + KbPdfTransaction( + page=7, + transaction_datetime="2026-06-18T09:00:00", + transaction_date="2026-06-18", + raw_summary="적요-민감메모", + counterparty="테스트상대", + withdrawal=42000, + deposit=0, + balance_after=123456, + remittance_memo="숨기고싶은메모", + branch="강남지점", + ) + ], + "validation": {"ok": True, "errors": []}, + } + monkeypatch.setattr(service, "parse_kb_bank_pdf", lambda file_path, password: parsed) + + imported = service.import_kb_bank_pdf("ignored.pdf", "pw", db_path=db_path) + assert imported["success"] is True + + listed = service.list_ledger_import_batches(source="kb_bank_pdf", db_path=db_path) + + assert listed["success"] is True + assert listed["count"] == 1 + batch = listed["import_batches"][0] + assert batch["memo"] == "" + assert batch["filename"] == _expected_filename(SECRET_SOURCE_PATH) + assert batch["provenance"] == { + "source": "kb_bank_pdf", + "parser": "kb_bank_pdf", + "import_batch_id": imported["import_batch_id"], + "source_hash": SOURCE_HASH, + "source_hash_prefix": SOURCE_HASH[:12], + } + rendered = json.dumps(batch, ensure_ascii=False) + assert SECRET_SOURCE_PATH not in rendered + assert "account_number_masked" not in rendered + assert "validation" not in rendered + + +def test_list_ledger_import_batches_operator_summary_exposes_masked_filename_and_validation(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": "/very/secret/home/alice/Documents/private/seoulpay.xlsx", + "source_hash": SOURCE_HASH, + "sheet_name": "결제내역", + "metadata": {"holder_name": "홍길동", "period": "20260601~20260620"}, + "transactions": [ + SeoulPayTransaction( + row_index=11, + channel="서울페이", + transaction_no="TRX-001", + transaction_datetime="2026-06-18 18:27:46", + transaction_date="2026-06-18", + merchant="땡겨요", + business_no="2028102637", + merchant_id="AF2402815444", + transaction_kind="온라인 결제", + payment_kind="결제", + amount=42000, + supply_amount=38182, + service_fee=0, + vat=3818, + tax_status="과세", + ) + ], + "validation": {"ok": True, "parsed_rows": 1, "errors": []}, + } + monkeypatch.setattr(service, "parse_seoulpay_xlsx", lambda file_path, password="": parsed) + + imported = service.import_seoulpay_xlsx("ignored.xlsx", db_path=db_path) + assert imported["success"] is True + + listed = service.list_ledger_import_batches( + source="seoulpay_xlsx", + provenance_exposure=EXPOSURE_LEVEL_OPERATOR_SUMMARY, + db_path=db_path, + ) + + assert listed["success"] is True + batch = listed["import_batches"][0] + assert batch["memo"] == "" + assert batch["provenance"] == { + "source": "seoulpay_xlsx", + "parser": "seoulpay_xlsx", + "filename": _expected_filename(parsed["file_path"]), + "import_batch_id": imported["import_batch_id"], + "sheet_name": "결제내역", + "source_hash": SOURCE_HASH, + "source_hash_prefix": SOURCE_HASH[:12], + "validation": {"ok": True, "parsed_rows": 1, "errors": []}, + } + rendered = json.dumps(batch, ensure_ascii=False) + assert parsed["file_path"] not in rendered + assert "holder_name" not in rendered + assert "period" not in rendered + + +def test_list_ledger_import_batches_requires_explicit_gate_for_debug(monkeypatch, tmp_path): + db_path = str(tmp_path / "ledger.sqlite3") + parsed = { + "file_path": SECRET_SOURCE_PATH, + "source_hash": SOURCE_HASH, + "page_count": 1, + "metadata": {"account_name": "생활비", "account_number_masked": "****5704", "holder_name": "홍길동", "final_balance": 123456}, + "validation": {"ok": True, "errors": []}, + "transactions": [ + KbPdfTransaction( + page=7, + transaction_datetime="2026-06-18T09:00:00", + transaction_date="2026-06-18", + raw_summary="적요-민감메모", + counterparty="테스트상대", + withdrawal=42000, + deposit=0, + balance_after=123456, + remittance_memo="숨기고싶은메모", + branch="강남지점", + ) + ], + } + monkeypatch.setattr(service, "parse_kb_bank_pdf", lambda file_path, password: parsed) + + imported = service.import_kb_bank_pdf("ignored.pdf", "pw", db_path=db_path) + assert imported["success"] is True + + denied = service.list_ledger_import_batches( + source="kb_bank_pdf", + provenance_exposure=EXPOSURE_LEVEL_DEBUG_GATED, + db_path=db_path, + ) + assert denied == { + "success": False, + "error_code": "forbidden", + "error": "debug-gated provenance requires explicit debug_provenance_gate=True", + } + + allowed = service.list_ledger_import_batches( + source="kb_bank_pdf", + provenance_exposure=EXPOSURE_LEVEL_DEBUG_GATED, + debug_provenance_gate=True, + db_path=db_path, + ) + assert allowed["success"] is True + assert allowed["import_batches"][0]["provenance"]["filename"] == _expected_filename(SECRET_SOURCE_PATH) + assert "debug" not in allowed["import_batches"][0]["provenance"] + + +def test_mcp_list_ledger_import_batches_passes_provenance_exposure(monkeypatch): + from jarca_ledger_mcp.adapters.mcp import server + + captured = {} + + def fake_list_ledger_import_batches(source, include_rolled_back, limit, provenance_exposure="minimal", debug_provenance_gate=False): + captured.update( + source=source, + include_rolled_back=include_rolled_back, + limit=limit, + provenance_exposure=provenance_exposure, + debug_provenance_gate=debug_provenance_gate, + ) + return {"success": True, "import_batches": [], "count": 0} + + monkeypatch.setattr(server.contracts, "list_ledger_import_batches", fake_list_ledger_import_batches) + + result = server.list_ledger_import_batches(source="kb_bank_pdf", include_rolled_back=False, limit=5) + assert result["success"] is True + assert captured == { + "source": "kb_bank_pdf", + "include_rolled_back": False, + "limit": 5, + "provenance_exposure": EXPOSURE_LEVEL_MINIMAL, + "debug_provenance_gate": False, + } + + result = server.list_ledger_import_batches( + source="seoulpay_xlsx", + include_rolled_back=True, + limit=3, + provenance_exposure=EXPOSURE_LEVEL_OPERATOR_SUMMARY, + debug_provenance_gate=True, + ) + assert result["success"] is True + assert captured == { + "source": "seoulpay_xlsx", + "include_rolled_back": True, + "limit": 3, + "provenance_exposure": EXPOSURE_LEVEL_OPERATOR_SUMMARY, + "debug_provenance_gate": True, + }