[verified] Canonicalize native review publication journals
This commit is contained in:
@@ -22,6 +22,10 @@ _BROKER_PATH = Path(__file__).resolve().parents[5] / 'lib/managed_lock_broker.py
|
||||
_broker_spec = importlib.util.spec_from_file_location('gitea_managed_lock_broker', _BROKER_PATH)
|
||||
_broker = importlib.util.module_from_spec(_broker_spec)
|
||||
_broker_spec.loader.exec_module(_broker)
|
||||
_JOURNAL_PATH = Path(__file__).resolve().with_name('publication_journal.py')
|
||||
_journal_spec = importlib.util.spec_from_file_location('gitea_publication_journal', _JOURNAL_PATH)
|
||||
_journal = importlib.util.module_from_spec(_journal_spec)
|
||||
_journal_spec.loader.exec_module(_journal)
|
||||
ValidationError = _support.ValidationError
|
||||
loads_json = _support.loads_json
|
||||
collect_pages = _support.collect_pages
|
||||
@@ -29,6 +33,249 @@ ORIGIN = 'https://gitea.jsfamily.quest'
|
||||
SCHEMA = 'gitea-agent.code-review-publication/v1'
|
||||
|
||||
|
||||
def canonical_journal_path(plan):
|
||||
"""Return the fixed, versioned path for this publication binding."""
|
||||
return _journal.canonical_path(plan, anchor_file=__file__, origin=ORIGIN,
|
||||
validate=validate_plan)
|
||||
|
||||
|
||||
def _expected_intent(plan, before):
|
||||
payload = build_payload(plan)
|
||||
return {'schema': 'gitea-agent.native-review-intent/v1',
|
||||
'fingerprint': hashlib.sha256(canonical(plan)).hexdigest(),
|
||||
'before_sha256': hashlib.sha256(canonical(before)).hexdigest(),
|
||||
'head': plan['target']['head'], 'event': payload['event'],
|
||||
'body_sha256': hashlib.sha256(payload['body'].encode()).hexdigest()}
|
||||
|
||||
|
||||
def _legacy_path(directory):
|
||||
import os, stat
|
||||
value = os.fspath(directory)
|
||||
if not isinstance(value, str) or not value.startswith('/') or any(
|
||||
part in ('', '.', '..') for part in value.split('/')[1:]):
|
||||
fail('unsafe legacy journal path')
|
||||
source = Path(value)
|
||||
try:
|
||||
_journal._check_ancestor_chain(source)
|
||||
st = source.lstat()
|
||||
except (OSError, ValueError):
|
||||
fail('unsafe legacy journal')
|
||||
if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700:
|
||||
fail('unsafe legacy journal')
|
||||
return source, (st.st_dev, st.st_ino)
|
||||
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _locked_legacy(directory):
|
||||
import fcntl, os, stat
|
||||
source, identity = _legacy_path(directory)
|
||||
fd = None
|
||||
try:
|
||||
# Never create a source mutex: registration preserves the entire source.
|
||||
fd = os.open(source / 'publication.lock', os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
st = os.fstat(fd)
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1:
|
||||
fail('unsafe legacy journal lock')
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
def check():
|
||||
_, current = _legacy_path(source)
|
||||
live = (source / 'publication.lock').lstat()
|
||||
if current != identity or (live.st_dev, live.st_ino) != (st.st_dev, st.st_ino):
|
||||
fail('legacy journal identity changed')
|
||||
check()
|
||||
yield source, check
|
||||
check()
|
||||
except ValidationError:
|
||||
raise
|
||||
except (OSError, ValueError):
|
||||
fail('legacy journal unavailable or unsafe')
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _capture_legacy(source):
|
||||
import stat
|
||||
records = {}; raw_records = {}; inventory = []
|
||||
allowed = {'before.json', 'intent.json', 'verified.json', 'publication.lock'}
|
||||
try:
|
||||
entries = list(source.iterdir())
|
||||
except OSError:
|
||||
fail('legacy journal artifact unavailable or unsafe')
|
||||
for entry in entries:
|
||||
if entry.name not in allowed:
|
||||
fail('unknown legacy journal artifact')
|
||||
try:
|
||||
st = entry.lstat()
|
||||
except OSError:
|
||||
fail('legacy journal artifact unavailable or unsafe')
|
||||
if entry.name == 'publication.lock':
|
||||
continue
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1:
|
||||
fail('unsafe legacy journal artifact')
|
||||
for name in ('before.json', 'intent.json', 'verified.json'):
|
||||
try:
|
||||
raw = _journal._secure_file(source / name, cap=16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except (OSError, ValueError):
|
||||
fail('legacy journal artifact unavailable or unsafe')
|
||||
try:
|
||||
records[name] = loads_json(raw.decode('utf-8'))
|
||||
raw_records[name] = raw
|
||||
except (UnicodeError, ValueError):
|
||||
fail('malformed legacy journal artifact')
|
||||
inventory.append({'name': name, 'size': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
framed = {'schema': 'gitea-agent.legacy-native-journal-digest/v1', 'files': inventory}
|
||||
return records, raw_records, framed, hashlib.sha256(canonical(framed)).hexdigest()
|
||||
|
||||
|
||||
def legacy_journal_digest(legacy_directory):
|
||||
"""Read-only digest of exact, named source bytes under its existing mutex."""
|
||||
with _locked_legacy(legacy_directory) as (source, check):
|
||||
_, _, _, digest_value = _capture_legacy(source)
|
||||
check()
|
||||
return digest_value
|
||||
|
||||
|
||||
def _assert_registry(entry, plan):
|
||||
try:
|
||||
_journal.assert_bound(entry, plan, anchor_file=__file__, origin=ORIGIN, validate=validate_plan)
|
||||
except (OSError, ValueError):
|
||||
fail('canonical journal binding unavailable or changed')
|
||||
|
||||
|
||||
def _check_registration(journal, plan):
|
||||
if journal.read('import.pending.json') is not None:
|
||||
fail('interrupted legacy registration')
|
||||
receipt = journal.read('registration.json')
|
||||
if receipt is None:
|
||||
return
|
||||
keys = {'schema', 'state', 'legacy_sha256', 'source_files', 'imported_files', 'before_sha256', 'intent_sha256'}
|
||||
if type(receipt) is not dict or set(receipt) != keys or receipt['schema'] != 'gitea-agent.native-review-import/v1' or receipt['state'] != 'complete':
|
||||
fail('interrupted or malformed legacy registration')
|
||||
files = receipt['source_files']
|
||||
if type(files) is not list or len(files) not in (2, 3):
|
||||
fail('malformed legacy registration file inventory')
|
||||
if [f.get('name') if type(f) is dict else None for f in files] not in (
|
||||
['before.json', 'intent.json'], ['before.json', 'intent.json', 'verified.json']):
|
||||
fail('malformed legacy registration file inventory')
|
||||
for f in files:
|
||||
if set(f) != {'name', 'size', 'sha256'} or type(f['size']) is not int or not 0 <= f['size'] <= (16 * 1024 * 1024 if f['name'] == 'before.json' else 1024 * 1024):
|
||||
fail('malformed legacy registration file inventory')
|
||||
digest(f['sha256'], 64)
|
||||
imported = receipt['imported_files']
|
||||
if type(imported) is not list or [f.get('source') if type(f) is dict else None for f in imported] not in (
|
||||
['before.json', 'intent.json'], ['before.json', 'intent.json', 'verified.json']):
|
||||
fail('malformed imported legacy file inventory')
|
||||
if [f['source'] for f in imported] != [f['name'] for f in files]:
|
||||
fail('incomplete imported legacy file inventory')
|
||||
for f in imported:
|
||||
if (set(f) != {'source', 'target', 'size', 'sha256'}
|
||||
or f['target'] != ('legacy-verified.json' if f['source'] == 'verified.json' else f['source'])
|
||||
or type(f['size']) is not int or f['size'] < 0):
|
||||
fail('malformed imported legacy file inventory')
|
||||
digest(f['sha256'], 64)
|
||||
source = next((row for row in files if row['name'] == f['source']), None)
|
||||
if source is None or source['size'] != f['size'] or source['sha256'] != f['sha256']:
|
||||
fail('legacy source/import receipt mismatch')
|
||||
for field in ('legacy_sha256', 'before_sha256', 'intent_sha256'):
|
||||
digest(receipt[field], 64)
|
||||
framed = {'schema': 'gitea-agent.legacy-native-journal-digest/v1', 'files': files}
|
||||
if hashlib.sha256(canonical(framed)).hexdigest() != receipt['legacy_sha256']:
|
||||
fail('legacy registration receipt mismatch')
|
||||
before = journal.read('before.json'); intent = journal.read('intent.json')
|
||||
if before is None or intent is None or hashlib.sha256(canonical(before)).hexdigest() != receipt['before_sha256'] or hashlib.sha256(canonical(intent)).hexdigest() != receipt['intent_sha256']:
|
||||
fail('legacy registration artifact mismatch')
|
||||
if canonical(intent) != canonical(_expected_intent(plan, before)):
|
||||
fail('legacy registration intent mismatch')
|
||||
for f in imported:
|
||||
try:
|
||||
raw = _journal._secure_file(Path(journal.path) / f['target'],
|
||||
cap=16 * 1024 * 1024 if f['source'] == 'before.json' else 1024 * 1024)
|
||||
except (OSError, ValueError):
|
||||
fail('legacy imported artifact unavailable or unsafe')
|
||||
if len(raw) != f['size'] or hashlib.sha256(raw).hexdigest() != f['sha256']:
|
||||
fail('legacy imported artifact integrity mismatch')
|
||||
try:
|
||||
loads_json(raw.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
fail('malformed legacy imported artifact')
|
||||
|
||||
|
||||
def register_legacy_journal(plan, legacy_directory, *, expected_legacy_sha256,
|
||||
api, lease, declaration=None):
|
||||
"""Explicit, local-only import; preserve source and never retry native POST."""
|
||||
validate_plan(plan)
|
||||
if type(expected_legacy_sha256) is not str or not re.fullmatch(r'[0-9a-f]{64}', expected_legacy_sha256):
|
||||
fail('invalid legacy journal digest')
|
||||
require_adoption(declaration, plan['target']['head'])
|
||||
check_lease(lease, plan)
|
||||
role_identity(read_api(api, 'identity'), plan['reviewer'])
|
||||
validate_snapshot(plan, read_api(api, 'snapshot', plan['target']))
|
||||
with _locked_legacy(legacy_directory) as (source, check):
|
||||
records, raw_records, framed, digest_value = _capture_legacy(source)
|
||||
if digest_value != expected_legacy_sha256:
|
||||
fail('legacy journal digest mismatch')
|
||||
before = records.get('before.json'); intent = records.get('intent.json')
|
||||
if type(before) is not dict or type(intent) is not dict:
|
||||
fail('legacy journal lacks durable intent')
|
||||
validate_snapshot(plan, before)
|
||||
if canonical(intent) != canonical(_expected_intent(plan, before)):
|
||||
fail('legacy journal intent mismatch')
|
||||
if 'verified.json' in records and type(records['verified.json']) is not dict:
|
||||
fail('invalid legacy verification artifact')
|
||||
check(); check_lease(lease, plan)
|
||||
target = canonical_journal_path(plan)
|
||||
try:
|
||||
entry = _journal.open_binding(target, plan, anchor_file=__file__, origin=ORIGIN,
|
||||
validate=validate_plan, require_new=True)
|
||||
except (OSError, ValueError):
|
||||
fail('canonical registration target unavailable or already exists')
|
||||
with Journal(entry) as journal:
|
||||
_assert_registry(entry, plan)
|
||||
if journal.read('import.pending.json') != {'schema': 'gitea-agent.native-review-import/v1', 'state': 'pending'}:
|
||||
fail('legacy registration bootstrap incomplete')
|
||||
if any(journal.read(name) is not None for name in ('before.json', 'intent.json', 'verified.json', 'registration.json')):
|
||||
fail('canonical journal already populated')
|
||||
receipt = {'schema': 'gitea-agent.native-review-import/v1', 'state': 'pending',
|
||||
'legacy_sha256': digest_value, 'source_files': framed['files'],
|
||||
'before_sha256': hashlib.sha256(canonical(before)).hexdigest(),
|
||||
'intent_sha256': hashlib.sha256(canonical(intent)).hexdigest()}
|
||||
# Durable pending receipt precedes copying any former attempt.
|
||||
journal.write('registration.json', receipt)
|
||||
imported_files = []
|
||||
for name in ('before.json', 'intent.json', 'verified.json'):
|
||||
if name in raw_records:
|
||||
target_name = 'legacy-verified.json' if name == 'verified.json' else name
|
||||
try:
|
||||
_journal._atomic_bytes(entry, target_name, raw_records[name])
|
||||
persisted = _journal._secure_file(entry / target_name,
|
||||
cap=16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024)
|
||||
except (OSError, ValueError):
|
||||
fail('legacy artifact import failed')
|
||||
if persisted != raw_records[name]:
|
||||
fail('legacy artifact integrity mismatch')
|
||||
imported_files.append({'source': name, 'target': target_name,
|
||||
'size': len(raw_records[name]),
|
||||
'sha256': hashlib.sha256(raw_records[name]).hexdigest()})
|
||||
receipt['imported_files'] = imported_files
|
||||
journal.write('registration.json', receipt)
|
||||
check()
|
||||
_assert_registry(entry, plan)
|
||||
receipt['state'] = 'complete'
|
||||
journal.write('registration.json', receipt)
|
||||
# The final receipt is durable before removing bootstrap's sentinel.
|
||||
import os
|
||||
journal.check_identity()
|
||||
os.unlink(entry / 'import.pending.json')
|
||||
_journal._fsync_dir(entry)
|
||||
return {'registered': True, 'journal_directory': str(entry), 'read_only': True}
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise ValidationError(message)
|
||||
|
||||
@@ -396,6 +643,21 @@ class Journal:
|
||||
if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700:
|
||||
fail('unsafe journal directory')
|
||||
self.fd = None
|
||||
self.identity = (st.st_dev, st.st_ino)
|
||||
|
||||
def check_identity(self):
|
||||
import os, stat
|
||||
try:
|
||||
_journal._check_ancestor_chain(self.path)
|
||||
st = self.path.lstat()
|
||||
if (st.st_dev, st.st_ino) != self.identity or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700:
|
||||
fail('journal directory identity changed')
|
||||
if self.fd is not None:
|
||||
held = os.fstat(self.fd); named = (self.path / 'publication.lock').lstat()
|
||||
if (held.st_dev, held.st_ino) != (named.st_dev, named.st_ino) or not stat.S_ISREG(named.st_mode) or named.st_nlink != 1 or named.st_uid != os.getuid() or stat.S_IMODE(named.st_mode) != 0o600:
|
||||
fail('journal mutex identity changed')
|
||||
except (OSError, ValueError):
|
||||
fail('journal identity unavailable or changed')
|
||||
|
||||
def __enter__(self):
|
||||
import fcntl, os, stat
|
||||
@@ -405,6 +667,10 @@ class Journal:
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1:
|
||||
fail('unsafe journal lock')
|
||||
fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
self.check_identity()
|
||||
except OSError:
|
||||
os.close(self.fd); self.fd = None
|
||||
fail('journal mutex unavailable or busy')
|
||||
except BaseException:
|
||||
os.close(self.fd); self.fd = None
|
||||
raise
|
||||
@@ -413,29 +679,38 @@ class Journal:
|
||||
def __exit__(self, *args):
|
||||
import os
|
||||
os.close(self.fd)
|
||||
self.fd = None
|
||||
|
||||
def read(self, name):
|
||||
import os, stat
|
||||
p = self.path / name
|
||||
self.check_identity()
|
||||
if name not in ('before.json', 'intent.json', 'verified.json', 'import.pending.json', 'registration.json'):
|
||||
fail('unsupported journal artifact')
|
||||
try:
|
||||
fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
cap = 16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024
|
||||
data = _journal._secure_file(self.path / name, cap=cap)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError:
|
||||
raise ValidationError('journal artifact unavailable') from None
|
||||
with os.fdopen(fd, 'r', encoding='utf-8') as artifact:
|
||||
st = os.fstat(artifact.fileno())
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1:
|
||||
fail('unsafe journal artifact')
|
||||
return loads_json(artifact.read())
|
||||
except (OSError, ValueError):
|
||||
fail('journal artifact unavailable or unsafe')
|
||||
try:
|
||||
value = loads_json(data.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
fail('malformed journal artifact')
|
||||
self.check_identity()
|
||||
return value
|
||||
|
||||
def write(self, name, value):
|
||||
import os, tempfile
|
||||
self.read(name) # Reject existing symlink/hardlink/foreign artifacts.
|
||||
data = canonical(value)
|
||||
cap = 16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024
|
||||
if len(data) > cap:
|
||||
fail('journal artifact exceeds bounds')
|
||||
fd, tmp = tempfile.mkstemp(dir=self.path, prefix='.native-review-')
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as out:
|
||||
out.write(canonical(value)); out.flush(); os.fsync(out.fileno())
|
||||
out.write(data); out.flush(); os.fsync(out.fileno())
|
||||
self.check_identity()
|
||||
os.replace(tmp, self.path / name)
|
||||
directory = os.open(self.path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
@@ -474,8 +749,8 @@ def check_lease(lease, plan):
|
||||
fail('unsupported lease check result')
|
||||
|
||||
|
||||
def publish(plan, baseline, api, lease, journal_directory, *, declaration=None):
|
||||
"""Use one trusted role transport and an already-held Issue lease; never retry POST."""
|
||||
def publish(plan, baseline, api, lease, journal_directory=None, *, declaration=None):
|
||||
"""Publish through the fixed canonical journal binding; never retry POST."""
|
||||
import copy
|
||||
validate_snapshot(plan, baseline)
|
||||
require_adoption(declaration, plan['target']['head'])
|
||||
@@ -486,8 +761,24 @@ def publish(plan, baseline, api, lease, journal_directory, *, declaration=None):
|
||||
fail('missing held Issue lease')
|
||||
fingerprint = hashlib.sha256(canonical(plan)).hexdigest()
|
||||
check_lease(lease, plan)
|
||||
with Journal(journal_directory) as journal:
|
||||
expected_directory = canonical_journal_path(plan)
|
||||
if journal_directory is not None:
|
||||
import os
|
||||
supplied = os.fspath(journal_directory)
|
||||
if type(supplied) is not str or supplied != str(expected_directory):
|
||||
fail('non-canonical journal path')
|
||||
# Authenticate the reviewer before claiming a registry binding.
|
||||
role_identity(read_api(api, 'identity'), plan['reviewer'])
|
||||
try:
|
||||
bound_directory = _journal.open_binding(expected_directory, plan, anchor_file=__file__, origin=ORIGIN, validate=validate_plan)
|
||||
except (OSError, ValueError):
|
||||
fail('canonical journal unavailable or conflicting binding')
|
||||
with Journal(bound_directory) as journal:
|
||||
_assert_registry(bound_directory, plan)
|
||||
_check_registration(journal, plan)
|
||||
intent = journal.read('intent.json')
|
||||
if intent is None and any(journal.read(name) is not None for name in ('before.json', 'verified.json')):
|
||||
raise UncertainPublication('incomplete canonical journal; do not retry POST')
|
||||
before = baseline
|
||||
if intent is not None:
|
||||
if type(intent) is not dict or intent.get('fingerprint') != fingerprint:
|
||||
@@ -540,6 +831,9 @@ def publish(plan, baseline, api, lease, journal_directory, *, declaration=None):
|
||||
'body_sha256': hashlib.sha256(payload['body'].encode()).hexdigest()})
|
||||
check_lease(lease, plan)
|
||||
require_adoption(declaration, plan['target']['head'])
|
||||
journal.check_identity()
|
||||
_assert_registry(bound_directory, plan)
|
||||
_check_registration(journal, plan)
|
||||
uncertain = False
|
||||
response_id = None
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
"""Canonical native-review publication journal registry."""
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import stat
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA = "gitea-agent.native-review-binding/v1"
|
||||
INTENT_SCHEMA = "gitea-agent.native-review-intent/v1"
|
||||
ROOT_NAME = "native-review-journals"
|
||||
_ROOT_PRESENCE_SCHEMA = "gitea-agent.native-review-root/v1"
|
||||
_ENTRY_PRESENCE_SCHEMA = "gitea-agent.native-review-entry/v1"
|
||||
_MAX_JSON = 16 * 1024 * 1024
|
||||
|
||||
|
||||
def _fail(message):
|
||||
raise ValueError(message)
|
||||
|
||||
|
||||
def _canonical(value):
|
||||
try:
|
||||
return json.dumps(value, sort_keys=True, ensure_ascii=False,
|
||||
allow_nan=False, separators=(",", ":"))
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
_fail("non-canonical JSON value")
|
||||
|
||||
|
||||
def _strict_load(data):
|
||||
def pairs(pairs):
|
||||
out = {}
|
||||
for key, value in pairs:
|
||||
if key in out:
|
||||
_fail("duplicate JSON key")
|
||||
out[key] = value
|
||||
return out
|
||||
try:
|
||||
return json.loads(data.decode("utf-8"), object_pairs_hook=pairs,
|
||||
parse_constant=lambda value: _fail("non-finite JSON"),
|
||||
parse_float=lambda value: float(value) if math.isfinite(float(value)) else _fail("non-finite JSON"))
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception:
|
||||
_fail("malformed journal artifact")
|
||||
|
||||
|
||||
def binding_for(plan, *, origin):
|
||||
t = plan["target"]
|
||||
return {"schema": SCHEMA, "origin": origin, "owner": t["owner"],
|
||||
"repository": t["repository"], "issue": t["issue"],
|
||||
"pull": t["pull"], "head": t["head"],
|
||||
"contract_sha256": plan["contract_sha256"],
|
||||
"reviewer_id": plan["reviewer"]["id"]}
|
||||
|
||||
|
||||
def plan_fingerprint(plan):
|
||||
return hashlib.sha256(_canonical(plan).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def root_for(anchor_file):
|
||||
harness = Path(anchor_file).resolve().parents[5]
|
||||
return harness / ".runtime" / ROOT_NAME
|
||||
|
||||
|
||||
def canonical_path(plan, *, anchor_file, origin, validate):
|
||||
validate(plan)
|
||||
binding = binding_for(plan, origin=origin)
|
||||
key = hashlib.sha256(_canonical(binding).encode("utf-8")).hexdigest()
|
||||
return root_for(anchor_file) / "v1" / key
|
||||
|
||||
|
||||
def _raw_path_safe(value):
|
||||
if isinstance(value, str):
|
||||
if not value.startswith("/"):
|
||||
_fail("unsafe journal path")
|
||||
parts = value.split("/")
|
||||
if any(part in ("", ".", "..") for part in parts[1:]):
|
||||
_fail("unsafe journal path")
|
||||
|
||||
|
||||
def _check_ancestor_chain(path, *, allow_missing_leaf=False):
|
||||
path = Path(path)
|
||||
if not path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
|
||||
_fail("unsafe journal path")
|
||||
current = Path(path.anchor)
|
||||
for index, part in enumerate(path.parts[1:]):
|
||||
current /= part
|
||||
try:
|
||||
st = current.lstat()
|
||||
except FileNotFoundError:
|
||||
if allow_missing_leaf and index == len(path.parts) - 2:
|
||||
return
|
||||
_fail("missing trusted journal ancestor")
|
||||
if stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode):
|
||||
_fail("unsafe journal ancestor")
|
||||
|
||||
|
||||
def _fsync_dir(path):
|
||||
fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _check_private_file(fd, *, what):
|
||||
st = os.fstat(fd)
|
||||
if (not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid()
|
||||
or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1):
|
||||
_fail("unsafe journal " + what)
|
||||
return st
|
||||
|
||||
|
||||
def _open_bootstrap(parent):
|
||||
path = Path(parent) / (ROOT_NAME + ".bootstrap.lock")
|
||||
try:
|
||||
fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_CLOEXEC, 0o600)
|
||||
except OSError:
|
||||
_fail("unsafe journal bootstrap lock")
|
||||
try:
|
||||
_check_private_file(fd, what="bootstrap lock")
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except (BlockingIOError, OSError) as exc:
|
||||
if getattr(exc, "errno", None) in (errno.EACCES, errno.EAGAIN):
|
||||
_fail("journal bootstrap busy")
|
||||
raise
|
||||
return fd
|
||||
except Exception:
|
||||
os.close(fd)
|
||||
raise
|
||||
|
||||
|
||||
def _presence_read(path, cap=4096):
|
||||
try:
|
||||
return _strict_load(_secure_file(path, cap=cap))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
def _atomic_bytes(directory, name, data, *, replace=False):
|
||||
directory = Path(directory)
|
||||
_check_ancestor_chain(directory)
|
||||
target = directory / name
|
||||
try:
|
||||
st = target.lstat()
|
||||
if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode) or st.st_nlink != 1:
|
||||
_fail("unsafe journal artifact")
|
||||
if not replace:
|
||||
_fail("journal artifact already exists")
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
fd, temp = tempfile.mkstemp(dir=directory, prefix=".native-review-")
|
||||
try:
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
out.write(data); out.flush(); os.fsync(out.fileno())
|
||||
if replace:
|
||||
os.replace(temp, target)
|
||||
else:
|
||||
try:
|
||||
os.link(temp, target)
|
||||
except FileExistsError:
|
||||
_fail("journal artifact already exists")
|
||||
os.unlink(temp)
|
||||
_fsync_dir(directory)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(temp)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
|
||||
def atomic_json(directory, name, value):
|
||||
_atomic_bytes(directory, name, _canonical(value).encode("utf-8"))
|
||||
|
||||
|
||||
def ensure_root(root, *, _lock_fd=None):
|
||||
root = Path(root)
|
||||
parent = root.parent
|
||||
_check_ancestor_chain(parent)
|
||||
lock_fd = _lock_fd if _lock_fd is not None else _open_bootstrap(parent)
|
||||
try:
|
||||
root_presence = parent / (root.name + ".presence")
|
||||
rp = _presence_read(root_presence)
|
||||
root_exists = False
|
||||
try:
|
||||
rst = root.lstat(); root_exists = True
|
||||
except FileNotFoundError:
|
||||
rst = None
|
||||
if rp is None:
|
||||
if root_exists:
|
||||
_fail("journal root presence anchor missing")
|
||||
root.mkdir(mode=0o700)
|
||||
rst = root.lstat()
|
||||
atomic_json(parent, root_presence.name, {"schema": _ROOT_PRESENCE_SCHEMA,
|
||||
"root": root.name + "/v1", "devino": [rst.st_dev, rst.st_ino]})
|
||||
else:
|
||||
if set(rp) != {"schema", "root", "devino"} or rp["schema"] != _ROOT_PRESENCE_SCHEMA:
|
||||
_fail("malformed journal root presence")
|
||||
if not root_exists or stat.S_ISLNK(rst.st_mode) or not stat.S_ISDIR(rst.st_mode):
|
||||
_fail("journal root was deleted")
|
||||
if [rst.st_dev, rst.st_ino] != rp["devino"]:
|
||||
_fail("journal root was replaced")
|
||||
rst = root.lstat()
|
||||
if (stat.S_ISLNK(rst.st_mode) or not stat.S_ISDIR(rst.st_mode)
|
||||
or rst.st_uid != os.getuid() or stat.S_IMODE(rst.st_mode) != 0o700):
|
||||
_fail("unsafe journal root")
|
||||
version = root / "v1"
|
||||
try: vst = version.lstat()
|
||||
except FileNotFoundError:
|
||||
version.mkdir(mode=0o700); _fsync_dir(root); vst = version.lstat()
|
||||
if (stat.S_ISLNK(vst.st_mode) or not stat.S_ISDIR(vst.st_mode)
|
||||
or vst.st_uid != os.getuid() or stat.S_IMODE(vst.st_mode) != 0o700):
|
||||
_fail("unsafe journal registry")
|
||||
return root
|
||||
finally:
|
||||
if _lock_fd is None:
|
||||
os.close(lock_fd)
|
||||
|
||||
|
||||
def _secure_file(path, *, cap=_MAX_JSON):
|
||||
_raw_path_safe(os.fspath(path))
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC)
|
||||
try:
|
||||
st = _check_private_file(fd, what="artifact")
|
||||
if st.st_size > cap:
|
||||
_fail("oversize journal artifact")
|
||||
chunks = []; total = 0
|
||||
while True:
|
||||
chunk = os.read(fd, min(65536, cap + 1 - total))
|
||||
if not chunk: break
|
||||
chunks.append(chunk); total += len(chunk)
|
||||
if total > cap: _fail("oversize journal artifact")
|
||||
return b"".join(chunks)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _entry_check(entry):
|
||||
st = Path(entry).lstat()
|
||||
if (stat.S_ISLNK(st.st_mode) or not stat.S_ISDIR(st.st_mode)
|
||||
or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700):
|
||||
_fail("unsafe journal binding")
|
||||
return st
|
||||
|
||||
|
||||
def _presence_path(entry):
|
||||
entry = Path(entry)
|
||||
return entry.parents[1].parent / (entry.name + ".presence")
|
||||
|
||||
|
||||
def open_binding(path, plan, *, anchor_file, origin, validate, require_new=False):
|
||||
_raw_path_safe(os.fspath(path))
|
||||
validate(plan)
|
||||
expected = canonical_path(plan, anchor_file=anchor_file, origin=origin, validate=validate)
|
||||
if Path(path) != expected or (isinstance(path, str) and path != str(expected)):
|
||||
_fail("non-canonical journal path")
|
||||
root = expected.parents[1]
|
||||
_check_ancestor_chain(root.parent, allow_missing_leaf=True)
|
||||
try:
|
||||
root.parent.mkdir(mode=0o700)
|
||||
_fsync_dir(root.parent.parent)
|
||||
except FileExistsError:
|
||||
pass
|
||||
_entry_check(root.parent)
|
||||
bootstrap = _open_bootstrap(root.parent)
|
||||
try:
|
||||
ensure_root(root, _lock_fd=bootstrap)
|
||||
entry = expected
|
||||
presence = _presence_path(entry)
|
||||
ep = _presence_read(presence)
|
||||
exists = entry.exists()
|
||||
if require_new and (exists or ep is not None):
|
||||
_fail("canonical journal entry already exists")
|
||||
if ep is not None:
|
||||
if set(ep) != {"schema", "entry", "devino", "fingerprint", "state"} or ep["schema"] != _ENTRY_PRESENCE_SCHEMA:
|
||||
_fail("malformed journal entry presence")
|
||||
if ep["state"] != "complete" or not exists:
|
||||
_fail("interrupted or deleted journal entry")
|
||||
st = _entry_check(entry)
|
||||
if [st.st_dev, st.st_ino] != ep["devino"]:
|
||||
_fail("journal entry was replaced")
|
||||
elif exists:
|
||||
_fail("journal entry presence anchor missing")
|
||||
else:
|
||||
entry.mkdir(mode=0o700); _fsync_dir(root)
|
||||
if require_new:
|
||||
atomic_json(entry, "import.pending.json", {
|
||||
"schema": "gitea-agent.native-review-import/v1",
|
||||
"state": "pending"})
|
||||
st = _entry_check(entry)
|
||||
pending = entry / "import.pending.json"
|
||||
if not require_new and pending.exists():
|
||||
_fail("interrupted legacy registration")
|
||||
binding = binding_for(plan, origin=origin)
|
||||
expected_record = {"schema": SCHEMA, "binding": binding, "fingerprint": plan_fingerprint(plan)}
|
||||
record_path = entry / "binding.json"
|
||||
try:
|
||||
existing = _strict_load(_secure_file(record_path, cap=1024 * 1024))
|
||||
except FileNotFoundError:
|
||||
existing = None
|
||||
if existing is not None and _canonical(existing) != _canonical(expected_record):
|
||||
_fail("binding record mismatch")
|
||||
if existing is None:
|
||||
if ep is not None:
|
||||
_fail("known journal binding record missing")
|
||||
if require_new is False and record_path.exists() and not record_path.is_file():
|
||||
_fail("unsafe journal binding record")
|
||||
atomic_json(entry, "binding.json", expected_record)
|
||||
if ep is None:
|
||||
_atomic_bytes(presence.parent, presence.name, _canonical({
|
||||
"schema": _ENTRY_PRESENCE_SCHEMA, "entry": entry.name,
|
||||
"devino": [st.st_dev, st.st_ino], "fingerprint": expected_record["fingerprint"],
|
||||
"state": "complete"}).encode())
|
||||
else:
|
||||
if ep["fingerprint"] != expected_record["fingerprint"]:
|
||||
_fail("entry binding fingerprint mismatch")
|
||||
return entry
|
||||
finally:
|
||||
os.close(bootstrap)
|
||||
|
||||
|
||||
def assert_bound(entry, plan, *, anchor_file, origin, validate):
|
||||
validate(plan)
|
||||
entry = Path(entry)
|
||||
expected = canonical_path(plan, anchor_file=anchor_file, origin=origin, validate=validate)
|
||||
if entry != expected:
|
||||
_fail("non-canonical journal path")
|
||||
root = expected.parents[1]
|
||||
_check_ancestor_chain(entry)
|
||||
_entry_check(root.parent); rst = _entry_check(root)
|
||||
_entry_check(root / 'v1'); st = _entry_check(entry)
|
||||
rp = _presence_read(root.parent / (root.name + ".presence"))
|
||||
expected_rp = {"schema": _ROOT_PRESENCE_SCHEMA, "root": root.name + "/v1",
|
||||
"devino": [rst.st_dev, rst.st_ino]}
|
||||
if _canonical(rp) != _canonical(expected_rp):
|
||||
_fail("journal root presence mismatch")
|
||||
expected_record = {"schema": SCHEMA, "binding": binding_for(plan, origin=origin), "fingerprint": plan_fingerprint(plan)}
|
||||
ep = _presence_read(_presence_path(entry))
|
||||
expected_ep = {"schema": _ENTRY_PRESENCE_SCHEMA, "entry": entry.name,
|
||||
"devino": [st.st_dev, st.st_ino], "fingerprint": expected_record["fingerprint"],
|
||||
"state": "complete"}
|
||||
if _canonical(ep) != _canonical(expected_ep):
|
||||
_fail("journal entry presence mismatch")
|
||||
record = _strict_load(_secure_file(entry / "binding.json", cap=1024 * 1024))
|
||||
if _canonical(record) != _canonical(expected_record):
|
||||
_fail("binding record mismatch")
|
||||
return entry
|
||||
|
||||
|
||||
def acquire_lock(entry):
|
||||
path = Path(entry) / "publication.lock"
|
||||
fd = None
|
||||
try:
|
||||
fd = os.open(path, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW | os.O_NONBLOCK | os.O_CLOEXEC, 0o600)
|
||||
_check_private_file(fd, what="lock")
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
if exc.errno in (errno.EACCES, errno.EAGAIN):
|
||||
_fail("journal lock busy")
|
||||
raise
|
||||
return fd
|
||||
except BaseException:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
raise
|
||||
|
||||
|
||||
def read_json(entry, name, cap=_MAX_JSON):
|
||||
try:
|
||||
return _strict_load(_secure_file(Path(entry) / name, cap=cap))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
Reference in New Issue
Block a user