gitea-impl-agent/jarca-ledger-mcp
1
0
Fork 0
forked from jsfamily/jarca-ledger-mcp
Code Pull Requests Activity

feat: enforce import provenance exposure policy (#2)

Browse Source 
Merge BAT-017 / REQ-021 implementation after delivery-runner implementation, independent QC, and passing GitHub Actions CI.
This commit is contained in:
genichin
2026-07-09 18:44:19 +09:00
committed by GitHub
parent c9231c2685
commit d9182f28f2
13 changed files with 1168 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jarca_ledger_mcp/adapters/mcp/server.py
+14 -2
View File
@@ -312,9 +312,21 @@ def import_toss_bank_pdf(file_path: str, account_id: str = "", skip_duplicates:
return contracts.import_toss_bank_pdf(file_path, account_id, skip_duplicates)
@mcp.tool()
def list_ledger_import_batches(source: str = "", include_rolled_back: bool = True, limit: int = 50) -> dict:
def list_ledger_import_batches(
source: str = "",
include_rolled_back: bool = True,
limit: int = 50,
provenance_exposure: str = "minimal",
debug_provenance_gate: bool = False,
) -> dict:
"""List CSV/PDF/API import batches."""
return contracts.list_ledger_import_batches(source, include_rolled_back, limit)
return contracts.list_ledger_import_batches(
source,
include_rolled_back,
limit,
provenance_exposure=provenance_exposure,
debug_provenance_gate=debug_provenance_gate,
)
@mcp.tool()
def rollback_ledger_import_batch(import_batch_id: str, hard_delete: bool = False) -> dict:
jarca_ledger_mcp/application/imports_service.py
+91 -14
View File
@@ -1,5 +1,13 @@
from __future__ import annotations
from ..provenance_policy import (
EXPOSURE_LEVEL_MINIMAL,
file_sha256,
redacted_source_file_label,
sanitize_csv_preview_rows,
sanitize_provenance_for_exposure,
)
from .shared import (
Any,
Path,
@@ -29,10 +37,9 @@ def preview_ledger_csv_import(file_path: str, source: str = "generic", db_path:
return fail("not_found", f"CSV file not found: {path}")
with path.open("r", encoding="utf-8-sig", newline="") as f:
rows=list(csv.DictReader(f))
preview=[]
for r in rows[:20]:
preview.append(r)
return ok(source=source, filename=str(path), row_count=len(rows), preview=preview, required_columns=["transaction_date/date", "amount", "type optional", "category optional", "merchant/description optional"])
source_hash = file_sha256(path)
preview = sanitize_csv_preview_rows(rows, limit=20)
return ok(source=source, filename=redacted_source_file_label(str(path), source_hash), source_hash=source_hash, row_count=len(rows), preview=preview, required_columns=["transaction_date/date", "amount", "type optional", "category optional", "merchant/description optional"])
def preview_kb_bank_pdf_import(file_path: str, password: str, preview_limit: int = 20, db_path: str | None = None) -> dict[str, Any]:
@@ -56,7 +63,7 @@ def preview_kb_bank_pdf_import(file_path: str, password: str, preview_limit: int
preview = [transaction_preview(tx, source_hash) for tx in transactions[: max(1, min(int(preview_limit), 100))]]
return ok(
source="kb_bank_pdf",
file_path=parsed["file_path"],
file_path=redacted_source_file_label(parsed["file_path"], source_hash),
source_hash=source_hash,
page_count=parsed["page_count"],
metadata=parsed["metadata"],
@@ -137,7 +144,7 @@ def import_kb_bank_pdf(file_path: str, password: str, account_id: str = "", skip
(
batch_id,
"kb_bank_pdf",
parsed["file_path"],
redacted_source_file_label(parsed["file_path"], source_hash),
ts,
0,
"active",
@@ -387,7 +394,7 @@ def preview_seoulpay_xlsx_import(file_path: str, password: str = "", preview_lim
preview = [seoulpay_transaction_preview(tx, source_hash) for tx in transactions[: max(1, min(int(preview_limit), 100))]]
return ok(
source="seoulpay_xlsx",
file_path=parsed["file_path"],
file_path=redacted_source_file_label(parsed["file_path"], source_hash),
source_hash=source_hash,
sheet_name=parsed["sheet_name"],
metadata=parsed["metadata"],
@@ -431,7 +438,7 @@ def import_seoulpay_xlsx(file_path: str, password: str = "", account_id: str = "
(
batch_id,
"seoulpay_xlsx",
parsed["file_path"],
redacted_source_file_label(parsed["file_path"], source_hash),
ts,
0,
"active",
@@ -553,7 +560,7 @@ def preview_toss_bank_pdf_import(file_path: str, preview_limit: int = 20, db_pat
item["amount"] += int(tx.amount)
return ok(
source="toss_bank_pdf",
file_path=parsed["file_path"],
file_path=redacted_source_file_label(parsed["file_path"], source_hash),
source_hash=source_hash,
page_count=parsed["page_count"],
metadata=parsed["metadata"],
@@ -597,7 +604,7 @@ def import_toss_bank_pdf(file_path: str, account_id: str = "", skip_duplicates:
(
batch_id,
"toss_bank_pdf",
parsed["file_path"],
redacted_source_file_label(parsed["file_path"], source_hash),
ts,
0,
"active",
@@ -786,7 +793,7 @@ def preview_toss_securities_pdf_import(file_path: str, preview_limit: int = 20,
item["amount"] += int(tx.amount)
return ok(
source="toss_securities_pdf",
file_path=parsed["file_path"],
file_path=redacted_source_file_label(parsed["file_path"], source_hash),
source_hash=source_hash,
page_count=parsed["page_count"],
metadata=parsed["metadata"],
@@ -827,7 +834,7 @@ def import_toss_securities_pdf(file_path: str, account_id: str = "", skip_duplic
(
batch_id,
"toss_securities_pdf",
parsed["file_path"],
redacted_source_file_label(parsed["file_path"], source_hash),
ts,
0,
"active",
@@ -916,7 +923,64 @@ def import_toss_securities_pdf(file_path: str, account_id: str = "", skip_duplic
)
def list_ledger_import_batches(source: str = "", include_rolled_back: bool = True, limit: int = 50, db_path: str | None = None) -> dict[str, Any]:
def list_ledger_import_batches(
source: str = "",
include_rolled_back: bool = True,
limit: int = 50,
provenance_exposure: str = EXPOSURE_LEVEL_MINIMAL,
debug_provenance_gate: bool = False,
db_path: str | None = None,
) -> dict[str, Any]:
return list_ledger_import_batches_with_provenance(
source=source,
include_rolled_back=include_rolled_back,
limit=limit,
provenance_exposure=provenance_exposure,
debug_provenance_gate=debug_provenance_gate,
db_path=db_path,
)
def _load_batch_memo(memo: Any) -> dict[str, Any]:
if isinstance(memo, dict):
return memo
if not isinstance(memo, str) or not memo.strip():
return {}
try:
parsed = json.loads(memo)
except json.JSONDecodeError:
return {}
return parsed if isinstance(parsed, dict) else {}
def _batch_provenance(row: dict[str, Any]) -> dict[str, Any]:
memo = _load_batch_memo(row.get("memo"))
provenance = {
"source": row.get("source") or "",
"parser": row.get("source") or "",
"filename": row.get("filename") or "",
"import_batch_id": row.get("id") or "",
"source_hash": memo.get("source_hash") or "",
"validation": memo.get("validation") if isinstance(memo.get("validation"), dict) else {},
}
if memo.get("sheet_name"):
provenance["sheet_name"] = memo["sheet_name"]
return provenance
def list_ledger_import_batches_with_provenance(
source: str = "",
include_rolled_back: bool = True,
limit: int = 50,
provenance_exposure: str = EXPOSURE_LEVEL_MINIMAL,
debug_provenance_gate: bool = False,
db_path: str | None = None,
) -> dict[str, Any]:
if provenance_exposure == "debug-gated" and not debug_provenance_gate:
return fail(
"forbidden",
"debug-gated provenance requires explicit debug_provenance_gate=True",
)
clauses=[]; params=[]
if source:
clauses.append("source=?"); params.append(source)
@@ -926,7 +990,20 @@ def list_ledger_import_batches(source: str = "", include_rolled_back: bool = Tru
params.append(max(1, min(int(limit), 500)))
with db(db_path) as conn:
rows=conn.execute(f"SELECT * FROM import_batches{where} ORDER BY imported_at DESC LIMIT ?", params).fetchall()
return ok(import_batches=[row_to_dict(r) for r in rows], count=len(rows))
try:
items = []
for row in rows:
item = row_to_dict(row)
provenance = _batch_provenance(item)
item["memo"] = ""
item["provenance"] = sanitize_provenance_for_exposure(
provenance,
provenance_exposure,
)
items.append(item)
except ValueError as exc:
return fail("validation_error", str(exc))
return ok(import_batches=items, count=len(items))
def rollback_ledger_import_batch(import_batch_id: str, hard_delete: bool = False, db_path: str | None = None) -> dict[str, Any]:
jarca_ledger_mcp/kb_bank_pdf.py
+4 -4
View File
@@ -1,13 +1,14 @@
from __future__ import annotations
import hashlib
import json
import re
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from .provenance_policy import allowlisted_json_payload
try:
import fitz # PyMuPDF
except ImportError: # pragma: no cover - exercised as runtime config error
@@ -55,7 +56,7 @@ class KbPdfTransaction:
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def raw_payload(self) -> str:
return json.dumps(
return allowlisted_json_payload(
{
"page": self.page,
"transaction_datetime": self.transaction_datetime,
@@ -67,8 +68,7 @@ class KbPdfTransaction:
"remittance_memo": self.remittance_memo,
"branch": self.branch,
},
ensure_ascii=False,
separators=(",", ":"),
allowed_keys=("page",),
)
jarca_ledger_mcp/provenance_policy.py
+357
View File
@@ -0,0 +1,357 @@
from __future__ import annotations
import hashlib
import json
import re
from pathlib import Path
from typing import Any, Iterable, Mapping
SOURCE_HASH_PREFIX_LENGTH = 12
CSV_PREVIEW_ALLOWED_FIELDS = (
"transaction_date",
"date",
"amount",
"type",
"category",
"subcategory",
"merchant",
"description",
)
EXPOSURE_LEVEL_MINIMAL = "minimal"
EXPOSURE_LEVEL_OPERATOR_SUMMARY = "operator-summary"
EXPOSURE_LEVEL_DEBUG_GATED = "debug-gated"
EXPOSURE_LEVELS = (
EXPOSURE_LEVEL_MINIMAL,
EXPOSURE_LEVEL_OPERATOR_SUMMARY,
EXPOSURE_LEVEL_DEBUG_GATED,
)
_FORBIDDEN_KEY_HINTS = (
"password",
"pwd",
"token",
"secret",
"credential",
"raw_document",
"raw_text",
"raw_dump",
"raw_payload_dump",
)
_PATH_KEY_HINTS = ("path", "filename", "filepath", "file_path", "source_path", "local_path")
_PATH_PREFIX_RE = re.compile(r"^[A-Za-z]:[\\/]|^(?:/|\\)")
_TOPLEVEL_FIELDS = {
EXPOSURE_LEVEL_MINIMAL: {
"source",
"source_hash",
"source_hash_prefix",
"parser",
"import_batch_id",
"external_id",
"dedupe_key",
},
EXPOSURE_LEVEL_OPERATOR_SUMMARY: {
"source",
"source_hash",
"source_hash_prefix",
"parser",
"filename",
"import_batch_id",
"external_id",
"dedupe_key",
"page",
"sheet",
"row_number",
"sheet_name",
"validation",
},
EXPOSURE_LEVEL_DEBUG_GATED: {
"source",
"source_hash",
"source_hash_prefix",
"parser",
"filename",
"import_batch_id",
"external_id",
"dedupe_key",
"page",
"sheet",
"row_number",
"sheet_name",
"validation",
"raw_payload",
"debug",
"diagnostics",
},
}
# Default field allowlists intentionally stay intentionally narrow and source-agnostic.
_RAW_PAYLOAD_ALLOWLIST = {
EXPOSURE_LEVEL_MINIMAL: frozenset(),
EXPOSURE_LEVEL_OPERATOR_SUMMARY: frozenset(
{
"row_index",
"page",
"sheet",
"transaction_date",
"transaction_datetime",
"type",
"amount",
"code",
"section",
"action",
"name",
"counterparty",
"merchant",
}
),
EXPOSURE_LEVEL_DEBUG_GATED: frozenset(
{
"row_index",
"page",
"sheet",
"transaction_date",
"transaction_datetime",
"type",
"amount",
"code",
"section",
"action",
"name",
"counterparty",
"merchant",
"transaction_no",
"business_no",
"merchant_id",
"channel",
"balance_after",
"quantity",
"unit_price",
"fee",
"tax",
"row_number",
"row_ref",
}
),
}
def source_hash_prefix(source_hash: str, length: int = SOURCE_HASH_PREFIX_LENGTH) -> str:
return (source_hash or "").strip()[:length]
def redacted_source_file_label(file_path: str, source_hash: str = "") -> str:
basename = Path(str(file_path or "")).name or "source"
token = source_hash_prefix(source_hash)
return f"{basename}#{token}" if token else basename
def file_sha256(file_path: str | Path) -> str:
return hashlib.sha256(Path(file_path).read_bytes()).hexdigest()
def allowlisted_json_payload(payload: Mapping[str, Any], allowed_keys: Iterable[str]) -> str:
cleaned = {
key: payload[key]
for key in allowed_keys
if key in payload and payload[key] not in (None, "", [], {})
}
return json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
def sanitize_csv_preview_rows(rows: list[Mapping[str, Any]], limit: int = 20) -> list[dict[str, Any]]:
preview: list[dict[str, Any]] = []
for row_number, row in enumerate(rows[: max(0, limit)], start=1):
item = {"row_number": row_number}
for key in CSV_PREVIEW_ALLOWED_FIELDS:
value = row.get(key)
if value not in (None, ""):
item[key] = value
preview.append(item)
return preview
def _is_forbidden_key(key: str) -> bool:
lower = key.lower()
return any(token in lower for token in _FORBIDDEN_KEY_HINTS)
def _is_path_key(key: str) -> bool:
lower = key.lower()
return any(lower.endswith(path_key) or lower == path_key for path_key in _PATH_KEY_HINTS)
def _is_full_path(value: str) -> bool:
if not value:
return False
return bool(_PATH_PREFIX_RE.search(value)) and ("/" in value or "\\" in value)
def _sanitize_mapping(
data: Mapping[str, Any],
*,
allowed_keys: set[str] | None,
source_hash: str,
) -> dict[str, Any]:
sanitized: dict[str, Any] = {}
for key, value in data.items():
key_name = str(key)
if allowed_keys is not None and key_name not in allowed_keys:
continue
if _is_forbidden_key(key_name):
continue
if _is_path_key(key_name) and isinstance(value, str) and _is_full_path(value):
sanitized[key_name] = redacted_source_file_label(value, source_hash)
continue
if isinstance(value, Mapping):
nested = _sanitize_mapping(value, allowed_keys=allowed_keys, source_hash=source_hash)
if nested:
sanitized[key_name] = nested
continue
if isinstance(value, list):
if value == []:
sanitized[key_name] = []
continue
nested_list: list[Any] = []
for item in value:
if isinstance(item, Mapping):
nested_item = _sanitize_mapping(item, allowed_keys=allowed_keys, source_hash=source_hash)
if nested_item:
nested_list.append(nested_item)
elif isinstance(item, str):
if _is_forbidden_key(item) or _is_full_path(item):
continue
nested_list.append(item)
else:
nested_list.append(item)
if nested_list:
sanitized[key_name] = nested_list
continue
if isinstance(value, str):
if _is_forbidden_key(value) or _is_full_path(value):
continue
if value is not None:
sanitized[key_name] = value
return sanitized
def _normalize_raw_payload(raw_payload: Any, exposure_level: str, source_hash: str) -> dict[str, Any]:
if isinstance(raw_payload, str):
try:
parsed_payload = json.loads(raw_payload)
except Exception:
return {}
else:
parsed_payload = raw_payload
if not isinstance(parsed_payload, Mapping):
return {}
allowlist = set(_RAW_PAYLOAD_ALLOWLIST[exposure_level])
if not allowlist:
return {}
return _sanitize_mapping(parsed_payload, allowed_keys=allowlist, source_hash=source_hash)
def sanitize_provenance(
provenance: Mapping[str, Any],
exposure_level: str = EXPOSURE_LEVEL_MINIMAL,
) -> dict[str, Any]:
if exposure_level not in EXPOSURE_LEVELS:
raise ValueError(f"unsupported exposure level: {exposure_level}")
source_hash = str(provenance.get("source_hash", ""))
source_hash_prefix_value = source_hash_prefix(source_hash)
allowed_fields = _TOPLEVEL_FIELDS[exposure_level]
sanitized: dict[str, Any] = {"source_hash_prefix": source_hash_prefix_value}
for key in sorted(allowed_fields):
if key == "source_hash_prefix":
sanitized[key] = source_hash_prefix_value
continue
if key not in provenance:
continue
if key in {"raw_payload", "debug", "diagnostics"} and key in provenance:
if key in {"raw_payload"}:
value = _normalize_raw_payload(
provenance[key],
exposure_level=exposure_level,
source_hash=source_hash,
)
else:
value = _sanitize_mapping(provenance[key], allowed_keys=None, source_hash=source_hash) if isinstance(provenance[key], Mapping) else provenance[key]
if value:
sanitized[key] = value
continue
value = provenance[key]
if _is_forbidden_key(str(key)):
continue
if isinstance(value, Mapping):
value = _sanitize_mapping(value, allowed_keys=None, source_hash=source_hash)
if not value:
continue
sanitized[key] = value
continue
if isinstance(value, list):
nested_list: list[Any] = []
for item in value:
if isinstance(item, Mapping):
item_value = _sanitize_mapping(item, allowed_keys=None, source_hash=source_hash)
if item_value:
nested_list.append(item_value)
elif isinstance(item, str):
if _is_forbidden_key(item) or _is_full_path(item):
continue
nested_list.append(item)
else:
nested_list.append(item)
if nested_list:
sanitized[key] = nested_list
continue
if _is_path_key(str(key)) and isinstance(value, str) and _is_full_path(value):
sanitized[key] = redacted_source_file_label(value, source_hash)
continue
if isinstance(value, str):
if _is_forbidden_key(value) or _is_full_path(value):
continue
if value is not None:
sanitized[key] = value
if "parser" not in sanitized and "source" in provenance:
sanitized["parser"] = provenance["source"]
if exposure_level != EXPOSURE_LEVEL_MINIMAL:
for path_key in ("file_path", "source_path", "local_path", "filename"):
raw_path = provenance.get(path_key)
if isinstance(raw_path, str) and _is_full_path(raw_path):
sanitized["filename"] = redacted_source_file_label(raw_path, source_hash)
break
# Keep raw document/text dumps suppressed even on debug output.
for forbidden_key in ("raw_document", "raw_text", "raw_dump", "raw_payload_dump"):
sanitized.pop(forbidden_key, None)
return sanitized
def sanitize_provenance_envelope(
provenance: Mapping[str, Any],
exposure_level: str = EXPOSURE_LEVEL_MINIMAL,
) -> dict[str, Any]:
return sanitize_provenance(provenance, exposure_level=exposure_level)
def sanitize_provenance_for_exposure(
provenance: Mapping[str, Any],
exposure_level: str = EXPOSURE_LEVEL_MINIMAL,
) -> dict[str, Any]:
return sanitize_provenance(provenance, exposure_level=exposure_level)
jarca_ledger_mcp/seoulpay_xlsx.py
+4 -4
View File
@@ -7,6 +7,8 @@ from datetime import datetime
from pathlib import Path
from typing import Any
from .provenance_policy import allowlisted_json_payload
@dataclass(frozen=True)
class SeoulPayTransaction:
@@ -38,9 +40,7 @@ class SeoulPayTransaction:
return f"seoulpay_xlsx:{source_hash}:{base}"
def raw_payload(self) -> str:
import json
return json.dumps(
return allowlisted_json_payload(
{
"row_index": self.row_index,
"channel": self.channel,
@@ -57,7 +57,7 @@ class SeoulPayTransaction:
"vat": self.vat,
"tax_status": self.tax_status,
},
ensure_ascii=False,
allowed_keys=("row_index",),
)
jarca_ledger_mcp/toss_bank_pdf.py
+4 -3
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import hashlib
import json
import re
from dataclasses import dataclass
from pathlib import Path
@@ -9,6 +8,8 @@ from typing import Any
import fitz
from .provenance_policy import allowlisted_json_payload
DATETIME_RE = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$")
AMOUNT_RE = re.compile(r"^[+-]?\d{1,3}(?:,\d{3})*$")
@@ -32,7 +33,7 @@ class TossBankTransaction:
return f"toss_bank_pdf:{source_hash}:{self.row_index}:{self.transaction_datetime}:{self.amount}"
def raw_payload(self) -> str:
return json.dumps(
return allowlisted_json_payload(
{
"row_index": self.row_index,
"transaction_datetime": self.transaction_datetime,
@@ -44,7 +45,7 @@ class TossBankTransaction:
"raw_summary": self.raw_summary,
"raw_lines": self.raw_lines,
},
ensure_ascii=False,
allowed_keys=("row_index",),
)
jarca_ledger_mcp/toss_securities_pdf.py
+4 -2
View File
@@ -9,6 +9,8 @@ from typing import Any
import fitz
from .provenance_policy import allowlisted_json_payload
DATE_RE = re.compile(r"^\d{4}\.\d{2}\.\d{2}$")
FOOTER_RE = re.compile(r"^\d+\s*/\s*\d+$")
NUM_RE = re.compile(r"^-?\d{1,3}(?:,\d{3})*(?:\.\d+)?$|^-?\d+(?:\.\d+)?$")
@@ -105,7 +107,7 @@ class TossSecuritiesTransaction:
return "tosssec:" + hashlib.sha256(raw.encode("utf-8")).hexdigest()
def raw_payload(self) -> str:
return json.dumps(
return allowlisted_json_payload(
{
"row_index": self.row_index,
"section": self.section,
@@ -123,7 +125,7 @@ class TossSecuritiesTransaction:
"raw_numbers": self.raw_numbers,
"raw_block": self.raw_block,
},
ensure_ascii=False,
allowed_keys=("row_index", "section", "quantity", "unit_price", "fee", "tax"),
)