feat: enforce import provenance exposure policy (#2)
Merge BAT-017 / REQ-021 implementation after delivery-runner implementation, independent QC, and passing GitHub Actions CI.
This commit is contained in:
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
SOURCE_HASH_PREFIX_LENGTH = 12
|
||||
CSV_PREVIEW_ALLOWED_FIELDS = (
|
||||
"transaction_date",
|
||||
"date",
|
||||
"amount",
|
||||
"type",
|
||||
"category",
|
||||
"subcategory",
|
||||
"merchant",
|
||||
"description",
|
||||
)
|
||||
|
||||
EXPOSURE_LEVEL_MINIMAL = "minimal"
|
||||
EXPOSURE_LEVEL_OPERATOR_SUMMARY = "operator-summary"
|
||||
EXPOSURE_LEVEL_DEBUG_GATED = "debug-gated"
|
||||
EXPOSURE_LEVELS = (
|
||||
EXPOSURE_LEVEL_MINIMAL,
|
||||
EXPOSURE_LEVEL_OPERATOR_SUMMARY,
|
||||
EXPOSURE_LEVEL_DEBUG_GATED,
|
||||
)
|
||||
|
||||
_FORBIDDEN_KEY_HINTS = (
|
||||
"password",
|
||||
"pwd",
|
||||
"token",
|
||||
"secret",
|
||||
"credential",
|
||||
"raw_document",
|
||||
"raw_text",
|
||||
"raw_dump",
|
||||
"raw_payload_dump",
|
||||
)
|
||||
_PATH_KEY_HINTS = ("path", "filename", "filepath", "file_path", "source_path", "local_path")
|
||||
_PATH_PREFIX_RE = re.compile(r"^[A-Za-z]:[\\/]|^(?:/|\\)")
|
||||
|
||||
_TOPLEVEL_FIELDS = {
|
||||
EXPOSURE_LEVEL_MINIMAL: {
|
||||
"source",
|
||||
"source_hash",
|
||||
"source_hash_prefix",
|
||||
"parser",
|
||||
"import_batch_id",
|
||||
"external_id",
|
||||
"dedupe_key",
|
||||
},
|
||||
EXPOSURE_LEVEL_OPERATOR_SUMMARY: {
|
||||
"source",
|
||||
"source_hash",
|
||||
"source_hash_prefix",
|
||||
"parser",
|
||||
"filename",
|
||||
"import_batch_id",
|
||||
"external_id",
|
||||
"dedupe_key",
|
||||
"page",
|
||||
"sheet",
|
||||
"row_number",
|
||||
"sheet_name",
|
||||
"validation",
|
||||
},
|
||||
EXPOSURE_LEVEL_DEBUG_GATED: {
|
||||
"source",
|
||||
"source_hash",
|
||||
"source_hash_prefix",
|
||||
"parser",
|
||||
"filename",
|
||||
"import_batch_id",
|
||||
"external_id",
|
||||
"dedupe_key",
|
||||
"page",
|
||||
"sheet",
|
||||
"row_number",
|
||||
"sheet_name",
|
||||
"validation",
|
||||
"raw_payload",
|
||||
"debug",
|
||||
"diagnostics",
|
||||
},
|
||||
}
|
||||
|
||||
# Default field allowlists intentionally stay intentionally narrow and source-agnostic.
|
||||
_RAW_PAYLOAD_ALLOWLIST = {
|
||||
EXPOSURE_LEVEL_MINIMAL: frozenset(),
|
||||
EXPOSURE_LEVEL_OPERATOR_SUMMARY: frozenset(
|
||||
{
|
||||
"row_index",
|
||||
"page",
|
||||
"sheet",
|
||||
"transaction_date",
|
||||
"transaction_datetime",
|
||||
"type",
|
||||
"amount",
|
||||
"code",
|
||||
"section",
|
||||
"action",
|
||||
"name",
|
||||
"counterparty",
|
||||
"merchant",
|
||||
}
|
||||
),
|
||||
EXPOSURE_LEVEL_DEBUG_GATED: frozenset(
|
||||
{
|
||||
"row_index",
|
||||
"page",
|
||||
"sheet",
|
||||
"transaction_date",
|
||||
"transaction_datetime",
|
||||
"type",
|
||||
"amount",
|
||||
"code",
|
||||
"section",
|
||||
"action",
|
||||
"name",
|
||||
"counterparty",
|
||||
"merchant",
|
||||
"transaction_no",
|
||||
"business_no",
|
||||
"merchant_id",
|
||||
"channel",
|
||||
"balance_after",
|
||||
"quantity",
|
||||
"unit_price",
|
||||
"fee",
|
||||
"tax",
|
||||
"row_number",
|
||||
"row_ref",
|
||||
}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def source_hash_prefix(source_hash: str, length: int = SOURCE_HASH_PREFIX_LENGTH) -> str:
|
||||
return (source_hash or "").strip()[:length]
|
||||
|
||||
|
||||
def redacted_source_file_label(file_path: str, source_hash: str = "") -> str:
|
||||
basename = Path(str(file_path or "")).name or "source"
|
||||
token = source_hash_prefix(source_hash)
|
||||
return f"{basename}#{token}" if token else basename
|
||||
|
||||
|
||||
def file_sha256(file_path: str | Path) -> str:
|
||||
return hashlib.sha256(Path(file_path).read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def allowlisted_json_payload(payload: Mapping[str, Any], allowed_keys: Iterable[str]) -> str:
|
||||
cleaned = {
|
||||
key: payload[key]
|
||||
for key in allowed_keys
|
||||
if key in payload and payload[key] not in (None, "", [], {})
|
||||
}
|
||||
return json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def sanitize_csv_preview_rows(rows: list[Mapping[str, Any]], limit: int = 20) -> list[dict[str, Any]]:
|
||||
preview: list[dict[str, Any]] = []
|
||||
for row_number, row in enumerate(rows[: max(0, limit)], start=1):
|
||||
item = {"row_number": row_number}
|
||||
for key in CSV_PREVIEW_ALLOWED_FIELDS:
|
||||
value = row.get(key)
|
||||
if value not in (None, ""):
|
||||
item[key] = value
|
||||
preview.append(item)
|
||||
return preview
|
||||
|
||||
|
||||
def _is_forbidden_key(key: str) -> bool:
|
||||
lower = key.lower()
|
||||
return any(token in lower for token in _FORBIDDEN_KEY_HINTS)
|
||||
|
||||
|
||||
def _is_path_key(key: str) -> bool:
|
||||
lower = key.lower()
|
||||
return any(lower.endswith(path_key) or lower == path_key for path_key in _PATH_KEY_HINTS)
|
||||
|
||||
|
||||
def _is_full_path(value: str) -> bool:
|
||||
if not value:
|
||||
return False
|
||||
return bool(_PATH_PREFIX_RE.search(value)) and ("/" in value or "\\" in value)
|
||||
|
||||
|
||||
def _sanitize_mapping(
|
||||
data: Mapping[str, Any],
|
||||
*,
|
||||
allowed_keys: set[str] | None,
|
||||
source_hash: str,
|
||||
) -> dict[str, Any]:
|
||||
sanitized: dict[str, Any] = {}
|
||||
for key, value in data.items():
|
||||
key_name = str(key)
|
||||
if allowed_keys is not None and key_name not in allowed_keys:
|
||||
continue
|
||||
if _is_forbidden_key(key_name):
|
||||
continue
|
||||
if _is_path_key(key_name) and isinstance(value, str) and _is_full_path(value):
|
||||
sanitized[key_name] = redacted_source_file_label(value, source_hash)
|
||||
continue
|
||||
if isinstance(value, Mapping):
|
||||
nested = _sanitize_mapping(value, allowed_keys=allowed_keys, source_hash=source_hash)
|
||||
if nested:
|
||||
sanitized[key_name] = nested
|
||||
continue
|
||||
if isinstance(value, list):
|
||||
if value == []:
|
||||
sanitized[key_name] = []
|
||||
continue
|
||||
|
||||
nested_list: list[Any] = []
|
||||
for item in value:
|
||||
if isinstance(item, Mapping):
|
||||
nested_item = _sanitize_mapping(item, allowed_keys=allowed_keys, source_hash=source_hash)
|
||||
if nested_item:
|
||||
nested_list.append(nested_item)
|
||||
elif isinstance(item, str):
|
||||
if _is_forbidden_key(item) or _is_full_path(item):
|
||||
continue
|
||||
nested_list.append(item)
|
||||
else:
|
||||
nested_list.append(item)
|
||||
if nested_list:
|
||||
sanitized[key_name] = nested_list
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
if _is_forbidden_key(value) or _is_full_path(value):
|
||||
continue
|
||||
if value is not None:
|
||||
sanitized[key_name] = value
|
||||
return sanitized
|
||||
|
||||
|
||||
def _normalize_raw_payload(raw_payload: Any, exposure_level: str, source_hash: str) -> dict[str, Any]:
|
||||
if isinstance(raw_payload, str):
|
||||
try:
|
||||
parsed_payload = json.loads(raw_payload)
|
||||
except Exception:
|
||||
return {}
|
||||
else:
|
||||
parsed_payload = raw_payload
|
||||
|
||||
if not isinstance(parsed_payload, Mapping):
|
||||
return {}
|
||||
|
||||
allowlist = set(_RAW_PAYLOAD_ALLOWLIST[exposure_level])
|
||||
if not allowlist:
|
||||
return {}
|
||||
|
||||
return _sanitize_mapping(parsed_payload, allowed_keys=allowlist, source_hash=source_hash)
|
||||
|
||||
|
||||
def sanitize_provenance(
|
||||
provenance: Mapping[str, Any],
|
||||
exposure_level: str = EXPOSURE_LEVEL_MINIMAL,
|
||||
) -> dict[str, Any]:
|
||||
if exposure_level not in EXPOSURE_LEVELS:
|
||||
raise ValueError(f"unsupported exposure level: {exposure_level}")
|
||||
|
||||
source_hash = str(provenance.get("source_hash", ""))
|
||||
source_hash_prefix_value = source_hash_prefix(source_hash)
|
||||
allowed_fields = _TOPLEVEL_FIELDS[exposure_level]
|
||||
|
||||
sanitized: dict[str, Any] = {"source_hash_prefix": source_hash_prefix_value}
|
||||
|
||||
for key in sorted(allowed_fields):
|
||||
if key == "source_hash_prefix":
|
||||
sanitized[key] = source_hash_prefix_value
|
||||
continue
|
||||
if key not in provenance:
|
||||
continue
|
||||
if key in {"raw_payload", "debug", "diagnostics"} and key in provenance:
|
||||
if key in {"raw_payload"}:
|
||||
value = _normalize_raw_payload(
|
||||
provenance[key],
|
||||
exposure_level=exposure_level,
|
||||
source_hash=source_hash,
|
||||
)
|
||||
else:
|
||||
value = _sanitize_mapping(provenance[key], allowed_keys=None, source_hash=source_hash) if isinstance(provenance[key], Mapping) else provenance[key]
|
||||
if value:
|
||||
sanitized[key] = value
|
||||
continue
|
||||
|
||||
value = provenance[key]
|
||||
if _is_forbidden_key(str(key)):
|
||||
continue
|
||||
if isinstance(value, Mapping):
|
||||
value = _sanitize_mapping(value, allowed_keys=None, source_hash=source_hash)
|
||||
if not value:
|
||||
continue
|
||||
sanitized[key] = value
|
||||
continue
|
||||
|
||||
if isinstance(value, list):
|
||||
nested_list: list[Any] = []
|
||||
for item in value:
|
||||
if isinstance(item, Mapping):
|
||||
item_value = _sanitize_mapping(item, allowed_keys=None, source_hash=source_hash)
|
||||
if item_value:
|
||||
nested_list.append(item_value)
|
||||
elif isinstance(item, str):
|
||||
if _is_forbidden_key(item) or _is_full_path(item):
|
||||
continue
|
||||
nested_list.append(item)
|
||||
else:
|
||||
nested_list.append(item)
|
||||
if nested_list:
|
||||
sanitized[key] = nested_list
|
||||
continue
|
||||
|
||||
if _is_path_key(str(key)) and isinstance(value, str) and _is_full_path(value):
|
||||
sanitized[key] = redacted_source_file_label(value, source_hash)
|
||||
continue
|
||||
|
||||
if isinstance(value, str):
|
||||
if _is_forbidden_key(value) or _is_full_path(value):
|
||||
continue
|
||||
|
||||
if value is not None:
|
||||
sanitized[key] = value
|
||||
|
||||
if "parser" not in sanitized and "source" in provenance:
|
||||
sanitized["parser"] = provenance["source"]
|
||||
|
||||
if exposure_level != EXPOSURE_LEVEL_MINIMAL:
|
||||
for path_key in ("file_path", "source_path", "local_path", "filename"):
|
||||
raw_path = provenance.get(path_key)
|
||||
if isinstance(raw_path, str) and _is_full_path(raw_path):
|
||||
sanitized["filename"] = redacted_source_file_label(raw_path, source_hash)
|
||||
break
|
||||
|
||||
# Keep raw document/text dumps suppressed even on debug output.
|
||||
for forbidden_key in ("raw_document", "raw_text", "raw_dump", "raw_payload_dump"):
|
||||
sanitized.pop(forbidden_key, None)
|
||||
|
||||
return sanitized
|
||||
|
||||
|
||||
def sanitize_provenance_envelope(
|
||||
provenance: Mapping[str, Any],
|
||||
exposure_level: str = EXPOSURE_LEVEL_MINIMAL,
|
||||
) -> dict[str, Any]:
|
||||
return sanitize_provenance(provenance, exposure_level=exposure_level)
|
||||
|
||||
|
||||
def sanitize_provenance_for_exposure(
|
||||
provenance: Mapping[str, Any],
|
||||
exposure_level: str = EXPOSURE_LEVEL_MINIMAL,
|
||||
) -> dict[str, Any]:
|
||||
return sanitize_provenance(provenance, exposure_level=exposure_level)
|
||||
Reference in New Issue
Block a user