246 lines
14 KiB
Python
246 lines
14 KiB
Python
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from harness.tests._support.paths import ROOT
|
|
|
|
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-rework/scripts/evidence_manifest.py"
|
|
|
|
|
|
def run_git(cwd, *args):
|
|
return subprocess.run(["git", *args], cwd=cwd, text=True, capture_output=True, check=True)
|
|
|
|
class ExternalEvidenceTests(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR', '/opt/data/cache/scratch'))
|
|
self.root = Path(self.tmp.name)
|
|
self.contract = self.root / 'harness/operations/repository-cache.md'
|
|
self.contract.parent.mkdir(parents=True)
|
|
self.contract.write_text('# Repository cache 운영 지침\n', encoding='utf-8')
|
|
self.repo = self.root / 'repositories/owner/project/worktrees/issue-1/checkout'
|
|
self.artifacts = self.repo.parent / 'verification/run-a'
|
|
self.repo.mkdir(parents=True)
|
|
self.artifacts.mkdir(parents=True)
|
|
for path in (self.root / 'repositories', *self.repo.parents):
|
|
if path == self.root:
|
|
break
|
|
path.chmod(448)
|
|
self.repo.chmod(448)
|
|
self.artifacts.parent.chmod(448)
|
|
self.artifacts.chmod(448)
|
|
run_git(self.repo, 'init', '-q')
|
|
run_git(self.repo, 'config', 'user.email', 'test@example.invalid')
|
|
run_git(self.repo, 'config', 'user.name', 'Test')
|
|
(self.repo / 'input.txt').write_text('hello\n')
|
|
run_git(self.repo, 'add', 'input.txt')
|
|
run_git(self.repo, 'commit', '-qm', 'initial')
|
|
self.plan = self.artifacts / 'evidence-plan.json'
|
|
self.plan.write_text(json.dumps({'hash_paths': ['input.txt'], 'commands': [{'id': 'check', 'argv': ['git', 'show', 'HEAD:input.txt'], 'timeout_seconds': 5, 'claims': ['output']}]}))
|
|
self.plan.chmod(384)
|
|
self.manifest = self.artifacts / 'evidence-manifest.json'
|
|
|
|
def tearDown(self):
|
|
self.tmp.cleanup()
|
|
|
|
def invoke(self, op, *args):
|
|
return subprocess.run([sys.executable, str(SCRIPT), op, '--contract-path', str(self.contract), '--repo-root', str(self.repo), '--artifact-root', str(self.artifacts), *map(str, args)], text=True, capture_output=True)
|
|
|
|
def test_ignored_source_input_is_rejected_for_run_and_validate(self):
|
|
(self.repo / '.gitignore').write_text('ignored.txt\n')
|
|
run_git(self.repo, 'add', '.gitignore')
|
|
run_git(self.repo, 'commit', '-qm', 'ignore fixture')
|
|
ignored = self.repo / 'ignored.txt'
|
|
ignored.write_text('unbound input\n')
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertIn('not clean', result.stderr)
|
|
self.assertFalse(self.manifest.exists())
|
|
ignored.unlink()
|
|
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
|
prior = self.manifest.read_bytes()
|
|
ignored.write_text('unbound input again\n')
|
|
result = self.invoke('validate', '--manifest', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertIn('not clean', result.stderr)
|
|
self.assertEqual(self.manifest.read_bytes(), prior)
|
|
|
|
def test_canonical_control_directory_modes_match_installed_harness(self):
|
|
self.root.chmod(504)
|
|
self.contract.parent.parent.chmod(511)
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 0)
|
|
self.assertEqual(self.root.stat().st_mode & 511, 504)
|
|
self.assertEqual(self.contract.parent.parent.stat().st_mode & 511, 511)
|
|
|
|
def test_same_filename_in_source_and_artifacts_is_not_a_collision(self):
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', 'input.txt')
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual((self.repo / 'input.txt').read_text(), 'hello\n')
|
|
self.assertEqual(self.invoke('validate', '--manifest', 'input.txt').returncode, 0)
|
|
|
|
def test_real_linked_detached_worktree_preserves_git_registration(self):
|
|
seed = self.root / 'seed'
|
|
self.repo.rename(seed)
|
|
bare = self.repo.parents[2] / 'repository.git'
|
|
subprocess.run(['git', 'clone', '--bare', str(seed), str(bare)], check=True, capture_output=True)
|
|
subprocess.run(['git', '--git-dir', str(bare), 'worktree', 'add', '--detach', str(self.repo), 'HEAD'], check=True, capture_output=True)
|
|
subprocess.run(['git', '--git-dir', str(bare), 'worktree', 'lock', str(self.repo)], check=True, capture_output=True)
|
|
before = run_git(self.repo, 'worktree', 'list', '--porcelain').stdout
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 0)
|
|
self.assertEqual(run_git(self.repo, 'worktree', 'list', '--porcelain').stdout, before)
|
|
self.assertNotEqual(run_git(self.repo, 'rev-parse', '--git-common-dir').stdout.strip(), '.git')
|
|
self.assertEqual(run_git(self.repo, 'status', '--porcelain').stdout, '')
|
|
|
|
def test_canonical_root_identity_and_traversal_denied_before_execution(self):
|
|
targets = (self.repo / 'docs/verification/run-a', self.root / 'scratch/run-a', self.repo.parent.parent / 'issue-2/verification/run-a', self.artifacts.parent / '../run-a')
|
|
for target in targets:
|
|
with self.subTest(target=str(target)):
|
|
result = self.invoke('run', '--artifact-root', target, '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertFalse(self.manifest.exists())
|
|
for filename in ('../escape.json', '/absolute.json', 'a/../escape.json', './plan.json', 'plan.json/'):
|
|
with self.subTest(filename=filename):
|
|
result = self.invoke('run', '--plan', filename, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
result = self.invoke('run', '--contract-path', self.root / 'wrong.md', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
|
|
def test_artifact_file_and_directory_symlinks_denied(self):
|
|
alias = self.artifacts / 'alias.json'
|
|
alias.symlink_to(self.plan.name)
|
|
result = self.invoke('run', '--plan', alias.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn('unsafe path', result.stderr)
|
|
outside = self.root / 'outside'
|
|
self.artifacts.rename(outside)
|
|
self.artifacts.symlink_to(outside, target_is_directory=True)
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn('symlink', result.stderr)
|
|
self.assertFalse((outside / self.manifest.name).exists())
|
|
|
|
def test_private_permissions_fail_closed_without_repair(self):
|
|
self.plan.chmod(420)
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertEqual(self.plan.stat().st_mode & 511, 420)
|
|
self.plan.chmod(384)
|
|
self.artifacts.chmod(493)
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertEqual(self.artifacts.stat().st_mode & 511, 493)
|
|
self.assertFalse(self.manifest.exists())
|
|
|
|
def test_wrong_uid_is_denied(self):
|
|
import importlib.util
|
|
from unittest.mock import patch
|
|
spec = importlib.util.spec_from_file_location('evidence_manifest_uid_test', SCRIPT)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
with patch.object(module.os, 'getuid', return_value=self.artifacts.stat().st_uid + 1):
|
|
with self.assertRaisesRegex(ValueError, 'ownership'):
|
|
module.safe_entry(self.artifacts, directory=True, mode=448)
|
|
|
|
def test_group_writable_cache_parent_requires_exclusive_role_membership(self):
|
|
import importlib.util
|
|
from types import SimpleNamespace
|
|
from unittest.mock import patch
|
|
spec = importlib.util.spec_from_file_location('evidence_manifest_group_test', SCRIPT)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
parent = self.root / 'repositories'
|
|
parent.chmod(509)
|
|
uid = parent.stat().st_uid
|
|
gid = parent.stat().st_gid
|
|
role = SimpleNamespace(pw_name='role', pw_uid=uid, pw_gid=gid)
|
|
foreign = SimpleNamespace(pw_name='foreign', pw_uid=uid + 1, pw_gid=gid)
|
|
with patch.object(module.grp, 'getgrgid', return_value=SimpleNamespace(gr_mem=[])), patch.object(module.pwd, 'getpwall', return_value=[role]), patch.object(module.pwd, 'getpwnam', return_value=role):
|
|
module.safe_repositories(parent)
|
|
with patch.object(module.grp, 'getgrgid', return_value=SimpleNamespace(gr_mem=['foreign'])), patch.object(module.pwd, 'getpwall', return_value=[role, foreign]), patch.object(module.pwd, 'getpwnam', side_effect=lambda name: role if name == 'role' else foreign):
|
|
with self.assertRaisesRegex(ValueError, 'exclusive'):
|
|
module.safe_repositories(parent)
|
|
with patch.object(module.grp, 'getgrgid', side_effect=KeyError('unresolved')):
|
|
with self.assertRaisesRegex(ValueError, 'unverifiable'):
|
|
module.safe_repositories(parent)
|
|
self.assertEqual(parent.stat().st_mode & 511, 509)
|
|
|
|
def test_no_clobber_rejects_rerun_before_command_execution(self):
|
|
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
|
prior = self.manifest.read_bytes()
|
|
obj = json.loads(self.plan.read_text())
|
|
marker = self.root / 'rerun-executed'
|
|
obj['commands'][0]['argv'] = [sys.executable, '-c', 'from pathlib import Path; Path(' + repr(str(marker)) + ").write_text('ran')"]
|
|
self.plan.write_text(json.dumps(obj))
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2)
|
|
self.assertIn('already exists', result.stderr)
|
|
self.assertEqual(self.manifest.read_bytes(), prior)
|
|
self.assertFalse(marker.exists())
|
|
|
|
def test_v2_rejects_forged_issue_binding_and_v1_normal_validation(self):
|
|
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
|
original = json.loads(self.manifest.read_text())
|
|
for replacement in (2, True, '1'):
|
|
obj = json.loads(json.dumps(original))
|
|
obj['artifact']['issue'] = replacement
|
|
self.manifest.write_text(json.dumps(obj, sort_keys=True, separators=(',', ':')) + '\n')
|
|
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 2)
|
|
original['schema'] = 'evidence-manifest/v1'
|
|
self.manifest.write_text(json.dumps(original, sort_keys=True, separators=(',', ':')) + '\n')
|
|
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 2)
|
|
|
|
def test_artifact_boundary_mutation_by_command_rejected_before_write(self):
|
|
obj = json.loads(self.plan.read_text())
|
|
obj['commands'][0]['argv'] = [sys.executable, '-c', 'from pathlib import Path; Path(' + repr(str(self.artifacts)) + ').chmod(0o755)']
|
|
self.plan.write_text(json.dumps(obj))
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertIn('permission', result.stderr)
|
|
self.assertFalse(self.manifest.exists())
|
|
|
|
def test_v2_stale_head_rejected_even_when_bound_file_is_unchanged(self):
|
|
self.assertEqual(self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name).returncode, 0)
|
|
(self.repo / 'other.txt').write_text('new source revision\n')
|
|
run_git(self.repo, 'add', 'other.txt')
|
|
run_git(self.repo, 'commit', '-qm', 'advance head')
|
|
result = self.invoke('validate', '--manifest', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertIn('source revision is not current HEAD', result.stderr)
|
|
|
|
def test_v2_rejects_source_symlink_alias(self):
|
|
(self.repo / 'alias.txt').symlink_to('input.txt')
|
|
run_git(self.repo, 'add', 'alias.txt')
|
|
run_git(self.repo, 'commit', '-qm', 'source symlink fixture')
|
|
obj = json.loads(self.plan.read_text())
|
|
obj['hash_paths'] = ['alias.txt']
|
|
self.plan.write_text(json.dumps(obj))
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 2, result.stderr)
|
|
self.assertIn('symlink', result.stderr)
|
|
self.assertFalse(self.manifest.exists())
|
|
|
|
def test_external_artifacts_leave_checkout_and_head_unchanged(self):
|
|
head = run_git(self.repo, 'rev-parse', 'HEAD').stdout.strip()
|
|
result = self.invoke('run', '--plan', self.plan.name, '--output', self.manifest.name)
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
data = json.loads(self.manifest.read_text())
|
|
self.assertEqual(data['schema'], 'evidence-manifest/v3')
|
|
self.assertEqual(data['source_revision'], head)
|
|
self.assertEqual(data['artifact']['run_id'], 'run-a')
|
|
self.assertEqual(data['artifact']['issue'], 1)
|
|
self.assertEqual(run_git(self.repo, 'status', '--porcelain').stdout, '')
|
|
self.assertEqual(run_git(self.repo, 'rev-parse', 'HEAD').stdout.strip(), head)
|
|
self.assertEqual(self.manifest.stat().st_mode & 511, 384)
|
|
self.assertEqual(self.invoke('validate', '--manifest', self.manifest.name).returncode, 0)
|