[verified] Canonicalize native review publication journals
This commit is contained in:
@@ -22,6 +22,10 @@ _BROKER_PATH = Path(__file__).resolve().parents[5] / 'lib/managed_lock_broker.py
|
||||
_broker_spec = importlib.util.spec_from_file_location('gitea_managed_lock_broker', _BROKER_PATH)
|
||||
_broker = importlib.util.module_from_spec(_broker_spec)
|
||||
_broker_spec.loader.exec_module(_broker)
|
||||
_JOURNAL_PATH = Path(__file__).resolve().with_name('publication_journal.py')
|
||||
_journal_spec = importlib.util.spec_from_file_location('gitea_publication_journal', _JOURNAL_PATH)
|
||||
_journal = importlib.util.module_from_spec(_journal_spec)
|
||||
_journal_spec.loader.exec_module(_journal)
|
||||
ValidationError = _support.ValidationError
|
||||
loads_json = _support.loads_json
|
||||
collect_pages = _support.collect_pages
|
||||
@@ -29,6 +33,249 @@ ORIGIN = 'https://gitea.jsfamily.quest'
|
||||
SCHEMA = 'gitea-agent.code-review-publication/v1'
|
||||
|
||||
|
||||
def canonical_journal_path(plan):
|
||||
"""Return the fixed, versioned path for this publication binding."""
|
||||
return _journal.canonical_path(plan, anchor_file=__file__, origin=ORIGIN,
|
||||
validate=validate_plan)
|
||||
|
||||
|
||||
def _expected_intent(plan, before):
|
||||
payload = build_payload(plan)
|
||||
return {'schema': 'gitea-agent.native-review-intent/v1',
|
||||
'fingerprint': hashlib.sha256(canonical(plan)).hexdigest(),
|
||||
'before_sha256': hashlib.sha256(canonical(before)).hexdigest(),
|
||||
'head': plan['target']['head'], 'event': payload['event'],
|
||||
'body_sha256': hashlib.sha256(payload['body'].encode()).hexdigest()}
|
||||
|
||||
|
||||
def _legacy_path(directory):
|
||||
import os, stat
|
||||
value = os.fspath(directory)
|
||||
if not isinstance(value, str) or not value.startswith('/') or any(
|
||||
part in ('', '.', '..') for part in value.split('/')[1:]):
|
||||
fail('unsafe legacy journal path')
|
||||
source = Path(value)
|
||||
try:
|
||||
_journal._check_ancestor_chain(source)
|
||||
st = source.lstat()
|
||||
except (OSError, ValueError):
|
||||
fail('unsafe legacy journal')
|
||||
if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700:
|
||||
fail('unsafe legacy journal')
|
||||
return source, (st.st_dev, st.st_ino)
|
||||
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _locked_legacy(directory):
|
||||
import fcntl, os, stat
|
||||
source, identity = _legacy_path(directory)
|
||||
fd = None
|
||||
try:
|
||||
# Never create a source mutex: registration preserves the entire source.
|
||||
fd = os.open(source / 'publication.lock', os.O_RDWR | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
st = os.fstat(fd)
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1:
|
||||
fail('unsafe legacy journal lock')
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
def check():
|
||||
_, current = _legacy_path(source)
|
||||
live = (source / 'publication.lock').lstat()
|
||||
if current != identity or (live.st_dev, live.st_ino) != (st.st_dev, st.st_ino):
|
||||
fail('legacy journal identity changed')
|
||||
check()
|
||||
yield source, check
|
||||
check()
|
||||
except ValidationError:
|
||||
raise
|
||||
except (OSError, ValueError):
|
||||
fail('legacy journal unavailable or unsafe')
|
||||
finally:
|
||||
if fd is not None:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _capture_legacy(source):
|
||||
import stat
|
||||
records = {}; raw_records = {}; inventory = []
|
||||
allowed = {'before.json', 'intent.json', 'verified.json', 'publication.lock'}
|
||||
try:
|
||||
entries = list(source.iterdir())
|
||||
except OSError:
|
||||
fail('legacy journal artifact unavailable or unsafe')
|
||||
for entry in entries:
|
||||
if entry.name not in allowed:
|
||||
fail('unknown legacy journal artifact')
|
||||
try:
|
||||
st = entry.lstat()
|
||||
except OSError:
|
||||
fail('legacy journal artifact unavailable or unsafe')
|
||||
if entry.name == 'publication.lock':
|
||||
continue
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_nlink != 1:
|
||||
fail('unsafe legacy journal artifact')
|
||||
for name in ('before.json', 'intent.json', 'verified.json'):
|
||||
try:
|
||||
raw = _journal._secure_file(source / name, cap=16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024)
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except (OSError, ValueError):
|
||||
fail('legacy journal artifact unavailable or unsafe')
|
||||
try:
|
||||
records[name] = loads_json(raw.decode('utf-8'))
|
||||
raw_records[name] = raw
|
||||
except (UnicodeError, ValueError):
|
||||
fail('malformed legacy journal artifact')
|
||||
inventory.append({'name': name, 'size': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
framed = {'schema': 'gitea-agent.legacy-native-journal-digest/v1', 'files': inventory}
|
||||
return records, raw_records, framed, hashlib.sha256(canonical(framed)).hexdigest()
|
||||
|
||||
|
||||
def legacy_journal_digest(legacy_directory):
|
||||
"""Read-only digest of exact, named source bytes under its existing mutex."""
|
||||
with _locked_legacy(legacy_directory) as (source, check):
|
||||
_, _, _, digest_value = _capture_legacy(source)
|
||||
check()
|
||||
return digest_value
|
||||
|
||||
|
||||
def _assert_registry(entry, plan):
|
||||
try:
|
||||
_journal.assert_bound(entry, plan, anchor_file=__file__, origin=ORIGIN, validate=validate_plan)
|
||||
except (OSError, ValueError):
|
||||
fail('canonical journal binding unavailable or changed')
|
||||
|
||||
|
||||
def _check_registration(journal, plan):
|
||||
if journal.read('import.pending.json') is not None:
|
||||
fail('interrupted legacy registration')
|
||||
receipt = journal.read('registration.json')
|
||||
if receipt is None:
|
||||
return
|
||||
keys = {'schema', 'state', 'legacy_sha256', 'source_files', 'imported_files', 'before_sha256', 'intent_sha256'}
|
||||
if type(receipt) is not dict or set(receipt) != keys or receipt['schema'] != 'gitea-agent.native-review-import/v1' or receipt['state'] != 'complete':
|
||||
fail('interrupted or malformed legacy registration')
|
||||
files = receipt['source_files']
|
||||
if type(files) is not list or len(files) not in (2, 3):
|
||||
fail('malformed legacy registration file inventory')
|
||||
if [f.get('name') if type(f) is dict else None for f in files] not in (
|
||||
['before.json', 'intent.json'], ['before.json', 'intent.json', 'verified.json']):
|
||||
fail('malformed legacy registration file inventory')
|
||||
for f in files:
|
||||
if set(f) != {'name', 'size', 'sha256'} or type(f['size']) is not int or not 0 <= f['size'] <= (16 * 1024 * 1024 if f['name'] == 'before.json' else 1024 * 1024):
|
||||
fail('malformed legacy registration file inventory')
|
||||
digest(f['sha256'], 64)
|
||||
imported = receipt['imported_files']
|
||||
if type(imported) is not list or [f.get('source') if type(f) is dict else None for f in imported] not in (
|
||||
['before.json', 'intent.json'], ['before.json', 'intent.json', 'verified.json']):
|
||||
fail('malformed imported legacy file inventory')
|
||||
if [f['source'] for f in imported] != [f['name'] for f in files]:
|
||||
fail('incomplete imported legacy file inventory')
|
||||
for f in imported:
|
||||
if (set(f) != {'source', 'target', 'size', 'sha256'}
|
||||
or f['target'] != ('legacy-verified.json' if f['source'] == 'verified.json' else f['source'])
|
||||
or type(f['size']) is not int or f['size'] < 0):
|
||||
fail('malformed imported legacy file inventory')
|
||||
digest(f['sha256'], 64)
|
||||
source = next((row for row in files if row['name'] == f['source']), None)
|
||||
if source is None or source['size'] != f['size'] or source['sha256'] != f['sha256']:
|
||||
fail('legacy source/import receipt mismatch')
|
||||
for field in ('legacy_sha256', 'before_sha256', 'intent_sha256'):
|
||||
digest(receipt[field], 64)
|
||||
framed = {'schema': 'gitea-agent.legacy-native-journal-digest/v1', 'files': files}
|
||||
if hashlib.sha256(canonical(framed)).hexdigest() != receipt['legacy_sha256']:
|
||||
fail('legacy registration receipt mismatch')
|
||||
before = journal.read('before.json'); intent = journal.read('intent.json')
|
||||
if before is None or intent is None or hashlib.sha256(canonical(before)).hexdigest() != receipt['before_sha256'] or hashlib.sha256(canonical(intent)).hexdigest() != receipt['intent_sha256']:
|
||||
fail('legacy registration artifact mismatch')
|
||||
if canonical(intent) != canonical(_expected_intent(plan, before)):
|
||||
fail('legacy registration intent mismatch')
|
||||
for f in imported:
|
||||
try:
|
||||
raw = _journal._secure_file(Path(journal.path) / f['target'],
|
||||
cap=16 * 1024 * 1024 if f['source'] == 'before.json' else 1024 * 1024)
|
||||
except (OSError, ValueError):
|
||||
fail('legacy imported artifact unavailable or unsafe')
|
||||
if len(raw) != f['size'] or hashlib.sha256(raw).hexdigest() != f['sha256']:
|
||||
fail('legacy imported artifact integrity mismatch')
|
||||
try:
|
||||
loads_json(raw.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
fail('malformed legacy imported artifact')
|
||||
|
||||
|
||||
def register_legacy_journal(plan, legacy_directory, *, expected_legacy_sha256,
|
||||
api, lease, declaration=None):
|
||||
"""Explicit, local-only import; preserve source and never retry native POST."""
|
||||
validate_plan(plan)
|
||||
if type(expected_legacy_sha256) is not str or not re.fullmatch(r'[0-9a-f]{64}', expected_legacy_sha256):
|
||||
fail('invalid legacy journal digest')
|
||||
require_adoption(declaration, plan['target']['head'])
|
||||
check_lease(lease, plan)
|
||||
role_identity(read_api(api, 'identity'), plan['reviewer'])
|
||||
validate_snapshot(plan, read_api(api, 'snapshot', plan['target']))
|
||||
with _locked_legacy(legacy_directory) as (source, check):
|
||||
records, raw_records, framed, digest_value = _capture_legacy(source)
|
||||
if digest_value != expected_legacy_sha256:
|
||||
fail('legacy journal digest mismatch')
|
||||
before = records.get('before.json'); intent = records.get('intent.json')
|
||||
if type(before) is not dict or type(intent) is not dict:
|
||||
fail('legacy journal lacks durable intent')
|
||||
validate_snapshot(plan, before)
|
||||
if canonical(intent) != canonical(_expected_intent(plan, before)):
|
||||
fail('legacy journal intent mismatch')
|
||||
if 'verified.json' in records and type(records['verified.json']) is not dict:
|
||||
fail('invalid legacy verification artifact')
|
||||
check(); check_lease(lease, plan)
|
||||
target = canonical_journal_path(plan)
|
||||
try:
|
||||
entry = _journal.open_binding(target, plan, anchor_file=__file__, origin=ORIGIN,
|
||||
validate=validate_plan, require_new=True)
|
||||
except (OSError, ValueError):
|
||||
fail('canonical registration target unavailable or already exists')
|
||||
with Journal(entry) as journal:
|
||||
_assert_registry(entry, plan)
|
||||
if journal.read('import.pending.json') != {'schema': 'gitea-agent.native-review-import/v1', 'state': 'pending'}:
|
||||
fail('legacy registration bootstrap incomplete')
|
||||
if any(journal.read(name) is not None for name in ('before.json', 'intent.json', 'verified.json', 'registration.json')):
|
||||
fail('canonical journal already populated')
|
||||
receipt = {'schema': 'gitea-agent.native-review-import/v1', 'state': 'pending',
|
||||
'legacy_sha256': digest_value, 'source_files': framed['files'],
|
||||
'before_sha256': hashlib.sha256(canonical(before)).hexdigest(),
|
||||
'intent_sha256': hashlib.sha256(canonical(intent)).hexdigest()}
|
||||
# Durable pending receipt precedes copying any former attempt.
|
||||
journal.write('registration.json', receipt)
|
||||
imported_files = []
|
||||
for name in ('before.json', 'intent.json', 'verified.json'):
|
||||
if name in raw_records:
|
||||
target_name = 'legacy-verified.json' if name == 'verified.json' else name
|
||||
try:
|
||||
_journal._atomic_bytes(entry, target_name, raw_records[name])
|
||||
persisted = _journal._secure_file(entry / target_name,
|
||||
cap=16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024)
|
||||
except (OSError, ValueError):
|
||||
fail('legacy artifact import failed')
|
||||
if persisted != raw_records[name]:
|
||||
fail('legacy artifact integrity mismatch')
|
||||
imported_files.append({'source': name, 'target': target_name,
|
||||
'size': len(raw_records[name]),
|
||||
'sha256': hashlib.sha256(raw_records[name]).hexdigest()})
|
||||
receipt['imported_files'] = imported_files
|
||||
journal.write('registration.json', receipt)
|
||||
check()
|
||||
_assert_registry(entry, plan)
|
||||
receipt['state'] = 'complete'
|
||||
journal.write('registration.json', receipt)
|
||||
# The final receipt is durable before removing bootstrap's sentinel.
|
||||
import os
|
||||
journal.check_identity()
|
||||
os.unlink(entry / 'import.pending.json')
|
||||
_journal._fsync_dir(entry)
|
||||
return {'registered': True, 'journal_directory': str(entry), 'read_only': True}
|
||||
|
||||
|
||||
def fail(message):
|
||||
raise ValidationError(message)
|
||||
|
||||
@@ -396,6 +643,21 @@ class Journal:
|
||||
if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700:
|
||||
fail('unsafe journal directory')
|
||||
self.fd = None
|
||||
self.identity = (st.st_dev, st.st_ino)
|
||||
|
||||
def check_identity(self):
|
||||
import os, stat
|
||||
try:
|
||||
_journal._check_ancestor_chain(self.path)
|
||||
st = self.path.lstat()
|
||||
if (st.st_dev, st.st_ino) != self.identity or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o700:
|
||||
fail('journal directory identity changed')
|
||||
if self.fd is not None:
|
||||
held = os.fstat(self.fd); named = (self.path / 'publication.lock').lstat()
|
||||
if (held.st_dev, held.st_ino) != (named.st_dev, named.st_ino) or not stat.S_ISREG(named.st_mode) or named.st_nlink != 1 or named.st_uid != os.getuid() or stat.S_IMODE(named.st_mode) != 0o600:
|
||||
fail('journal mutex identity changed')
|
||||
except (OSError, ValueError):
|
||||
fail('journal identity unavailable or changed')
|
||||
|
||||
def __enter__(self):
|
||||
import fcntl, os, stat
|
||||
@@ -405,6 +667,10 @@ class Journal:
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1:
|
||||
fail('unsafe journal lock')
|
||||
fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
self.check_identity()
|
||||
except OSError:
|
||||
os.close(self.fd); self.fd = None
|
||||
fail('journal mutex unavailable or busy')
|
||||
except BaseException:
|
||||
os.close(self.fd); self.fd = None
|
||||
raise
|
||||
@@ -413,29 +679,38 @@ class Journal:
|
||||
def __exit__(self, *args):
|
||||
import os
|
||||
os.close(self.fd)
|
||||
self.fd = None
|
||||
|
||||
def read(self, name):
|
||||
import os, stat
|
||||
p = self.path / name
|
||||
self.check_identity()
|
||||
if name not in ('before.json', 'intent.json', 'verified.json', 'import.pending.json', 'registration.json'):
|
||||
fail('unsupported journal artifact')
|
||||
try:
|
||||
fd = os.open(p, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
cap = 16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024
|
||||
data = _journal._secure_file(self.path / name, cap=cap)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError:
|
||||
raise ValidationError('journal artifact unavailable') from None
|
||||
with os.fdopen(fd, 'r', encoding='utf-8') as artifact:
|
||||
st = os.fstat(artifact.fileno())
|
||||
if not stat.S_ISREG(st.st_mode) or st.st_uid != os.getuid() or stat.S_IMODE(st.st_mode) != 0o600 or st.st_nlink != 1:
|
||||
fail('unsafe journal artifact')
|
||||
return loads_json(artifact.read())
|
||||
except (OSError, ValueError):
|
||||
fail('journal artifact unavailable or unsafe')
|
||||
try:
|
||||
value = loads_json(data.decode('utf-8'))
|
||||
except (UnicodeError, ValueError):
|
||||
fail('malformed journal artifact')
|
||||
self.check_identity()
|
||||
return value
|
||||
|
||||
def write(self, name, value):
|
||||
import os, tempfile
|
||||
self.read(name) # Reject existing symlink/hardlink/foreign artifacts.
|
||||
data = canonical(value)
|
||||
cap = 16 * 1024 * 1024 if name == 'before.json' else 1024 * 1024
|
||||
if len(data) > cap:
|
||||
fail('journal artifact exceeds bounds')
|
||||
fd, tmp = tempfile.mkstemp(dir=self.path, prefix='.native-review-')
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as out:
|
||||
out.write(canonical(value)); out.flush(); os.fsync(out.fileno())
|
||||
out.write(data); out.flush(); os.fsync(out.fileno())
|
||||
self.check_identity()
|
||||
os.replace(tmp, self.path / name)
|
||||
directory = os.open(self.path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
@@ -474,8 +749,8 @@ def check_lease(lease, plan):
|
||||
fail('unsupported lease check result')
|
||||
|
||||
|
||||
def publish(plan, baseline, api, lease, journal_directory, *, declaration=None):
|
||||
"""Use one trusted role transport and an already-held Issue lease; never retry POST."""
|
||||
def publish(plan, baseline, api, lease, journal_directory=None, *, declaration=None):
|
||||
"""Publish through the fixed canonical journal binding; never retry POST."""
|
||||
import copy
|
||||
validate_snapshot(plan, baseline)
|
||||
require_adoption(declaration, plan['target']['head'])
|
||||
@@ -486,8 +761,24 @@ def publish(plan, baseline, api, lease, journal_directory, *, declaration=None):
|
||||
fail('missing held Issue lease')
|
||||
fingerprint = hashlib.sha256(canonical(plan)).hexdigest()
|
||||
check_lease(lease, plan)
|
||||
with Journal(journal_directory) as journal:
|
||||
expected_directory = canonical_journal_path(plan)
|
||||
if journal_directory is not None:
|
||||
import os
|
||||
supplied = os.fspath(journal_directory)
|
||||
if type(supplied) is not str or supplied != str(expected_directory):
|
||||
fail('non-canonical journal path')
|
||||
# Authenticate the reviewer before claiming a registry binding.
|
||||
role_identity(read_api(api, 'identity'), plan['reviewer'])
|
||||
try:
|
||||
bound_directory = _journal.open_binding(expected_directory, plan, anchor_file=__file__, origin=ORIGIN, validate=validate_plan)
|
||||
except (OSError, ValueError):
|
||||
fail('canonical journal unavailable or conflicting binding')
|
||||
with Journal(bound_directory) as journal:
|
||||
_assert_registry(bound_directory, plan)
|
||||
_check_registration(journal, plan)
|
||||
intent = journal.read('intent.json')
|
||||
if intent is None and any(journal.read(name) is not None for name in ('before.json', 'verified.json')):
|
||||
raise UncertainPublication('incomplete canonical journal; do not retry POST')
|
||||
before = baseline
|
||||
if intent is not None:
|
||||
if type(intent) is not dict or intent.get('fingerprint') != fingerprint:
|
||||
@@ -540,6 +831,9 @@ def publish(plan, baseline, api, lease, journal_directory, *, declaration=None):
|
||||
'body_sha256': hashlib.sha256(payload['body'].encode()).hexdigest()})
|
||||
check_lease(lease, plan)
|
||||
require_adoption(declaration, plan['target']['head'])
|
||||
journal.check_identity()
|
||||
_assert_registry(bound_directory, plan)
|
||||
_check_registration(journal, plan)
|
||||
uncertain = False
|
||||
response_id = None
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user