test(harness): reorganize regression suites and preserve contract coverage
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
"""Read-only recovery must never repeat an uncertain delivery mutation."""
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
|
||||
PATH = ROOT / 'harness/agents/impl/skills/gitea-issue-implementation/scripts/implementation_delivery.py'
|
||||
|
||||
|
||||
class ReadOnlyAdapter:
|
||||
def __init__(self, head):
|
||||
self.head = head
|
||||
self.calls = []
|
||||
self.proof = True
|
||||
|
||||
def push(self, head):
|
||||
raise AssertionError('reconciliation must not push')
|
||||
|
||||
def create_or_reuse_pr(self, head):
|
||||
raise AssertionError('reconciliation must not create or patch')
|
||||
|
||||
def read_remote_head(self):
|
||||
self.calls.append('branch')
|
||||
return self.head
|
||||
|
||||
def read_pr(self, number):
|
||||
self.calls.append('pr')
|
||||
return {'number': number, 'head': self.head, 'state': 'open'}
|
||||
|
||||
def verify_reconciliation(self, head, number):
|
||||
self.calls.append('complete_intent')
|
||||
return self.proof
|
||||
|
||||
|
||||
class ReconciliationTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
spec = importlib.util.spec_from_file_location('delivery_reconciliation_under_test', PATH)
|
||||
cls.module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(cls.module)
|
||||
|
||||
def test_read_only_recovery_returns_exact_state_without_repeated_mutation(self):
|
||||
adapter = ReadOnlyAdapter('a' * 40)
|
||||
result = self.module.reconcile_implementation(adapter, head=adapter.head, number=19, declaration={'mode': 'none'})
|
||||
self.assertEqual(result, {'remote_head': adapter.head, 'pr': {'number': 19, 'head': adapter.head, 'state': 'open'}})
|
||||
self.assertEqual(adapter.calls, ['branch', 'pr', 'complete_intent'])
|
||||
|
||||
def test_missing_declaration_blocks_all_reads(self):
|
||||
adapter = ReadOnlyAdapter('a' * 40)
|
||||
with self.assertRaises(self.module.AdoptionError):
|
||||
self.module.reconcile_implementation(adapter, head=adapter.head, number=19, declaration=None)
|
||||
self.assertEqual(adapter.calls, [])
|
||||
|
||||
def test_invalid_identifiers_block_all_reads(self):
|
||||
for head, number in [('bad', 19), ('a' * 40, True), ('a' * 40, 0)]:
|
||||
adapter = ReadOnlyAdapter('a' * 40)
|
||||
with self.subTest(head=head, number=number), self.assertRaises(self.module.AdoptionError):
|
||||
self.module.reconcile_implementation(adapter, head=head, number=number, declaration={'mode': 'none'})
|
||||
self.assertEqual(adapter.calls, [])
|
||||
|
||||
def test_branch_mismatch_blocks_pr_and_proof_reads(self):
|
||||
adapter = ReadOnlyAdapter('b' * 40)
|
||||
with self.assertRaises(self.module.AdoptionError):
|
||||
self.module.reconcile_implementation(adapter, head='a' * 40, number=19, declaration={'mode': 'none'})
|
||||
self.assertEqual(adapter.calls, ['branch'])
|
||||
|
||||
def test_body_snapshot_scope_or_collection_failure_cannot_complete(self):
|
||||
for proof in [False, None, 1, {}, 'yes']:
|
||||
adapter = ReadOnlyAdapter('a' * 40)
|
||||
adapter.proof = proof
|
||||
with self.subTest(proof=proof), self.assertRaises(self.module.AdoptionError):
|
||||
self.module.reconcile_implementation(adapter, head=adapter.head, number=19, declaration={'mode': 'none'})
|
||||
self.assertEqual(adapter.calls, ['branch', 'pr', 'complete_intent'])
|
||||
|
||||
def test_loopback_committed_write_with_bad_ack_recovers_read_only(self):
|
||||
import json
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from urllib.request import Request, urlopen
|
||||
spec = importlib.util.spec_from_file_location('response_reconciliation_integration', PATH.with_name('delivery_response.py'))
|
||||
assert spec is not None and spec.loader is not None
|
||||
response = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(response)
|
||||
head = 'a' * 40
|
||||
state = {'writes': 0, 'body': '', 'head': head, 'number': 19, 'state': 'open',
|
||||
'base': 'main', 'repository': 'owner/repo', 'author': 6,
|
||||
'issue': {'body': 'approved', 'updated_at': 'pinned', 'comments': []},
|
||||
'files': ['one.py'], 'commits': [head], 'collections_complete': True}
|
||||
expected_issue = dict(state['issue'])
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_PATCH(self):
|
||||
state['writes'] += 1
|
||||
state['body'] = self.rfile.read(int(self.headers['Content-Length'])).decode()
|
||||
self.send_response(201)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{') # Committed, but malformed acknowledgement.
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(state).encode())
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
server = HTTPServer(('127.0.0.1', 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
url = 'http://127.0.0.1:' + str(server.server_port)
|
||||
def send(method='GET'):
|
||||
request = Request(url, data=b'intended' if method == 'PATCH' else None, method=method)
|
||||
with urlopen(request, timeout=2) as reply:
|
||||
return reply.status, reply.read(response.MAX_RESPONSE_BYTES + 1)
|
||||
def read():
|
||||
return response.deliver_once(send, expected_status=200)
|
||||
class Adapter(ReadOnlyAdapter):
|
||||
def read_remote_head(self):
|
||||
return read()['head']
|
||||
def read_pr(self, number):
|
||||
return read()
|
||||
def verify_reconciliation(self, head, number):
|
||||
actual = read()
|
||||
return (actual['body'] == 'intended' and actual['head'] == head
|
||||
and actual['number'] == number and actual['state'] == 'open'
|
||||
and actual['base'] == 'main' and actual['repository'] == 'owner/repo'
|
||||
and actual['author'] == 6 and actual['issue'] == expected_issue
|
||||
and actual['files'] == ['one.py'] and actual['commits'] == [head]
|
||||
and actual['collections_complete'] is True)
|
||||
try:
|
||||
with self.assertRaises(response.AcknowledgementUncertainError):
|
||||
response.deliver_once(lambda: send('PATCH'), expected_status=201)
|
||||
result = self.module.reconcile_implementation(Adapter(head), head=head, number=19, declaration={'mode': 'none'})
|
||||
self.assertEqual(result['remote_head'], head)
|
||||
for key, bad in [('body', 'competing change'), ('issue', {}), ('collections_complete', False)]:
|
||||
saved = state[key]
|
||||
state[key] = bad
|
||||
with self.subTest(key=key), self.assertRaises(self.module.AdoptionError):
|
||||
self.module.reconcile_implementation(Adapter(head), head=head, number=19, declaration={'mode': 'none'})
|
||||
state[key] = saved
|
||||
self.assertEqual(state['writes'], 1)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=2)
|
||||
self.assertFalse(thread.is_alive())
|
||||
|
||||
def test_closed_pr_cannot_complete(self):
|
||||
adapter = ReadOnlyAdapter('a' * 40)
|
||||
adapter.read_pr = lambda number: {'number': number, 'head': adapter.head, 'state': 'closed'}
|
||||
with self.assertRaises(self.module.AdoptionError):
|
||||
self.module.reconcile_implementation(adapter, head=adapter.head, number=19, declaration={'mode': 'none'})
|
||||
self.assertNotIn('complete_intent', adapter.calls)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,158 @@
|
||||
import json
|
||||
import threading
|
||||
import unittest
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
sys.path.insert(0, str(ROOT / 'harness/agents/impl/skills/gitea-issue-implementation/scripts'))
|
||||
import delivery_response as dr
|
||||
|
||||
|
||||
def make_handler():
|
||||
class FakeHandler(BaseHTTPRequestHandler):
|
||||
status = 200
|
||||
body = b'{"ok":true}'
|
||||
calls = 0
|
||||
|
||||
def do_GET(self): self.respond()
|
||||
def do_POST(self): self.respond()
|
||||
def do_PATCH(self): self.respond()
|
||||
|
||||
def respond(self):
|
||||
type(self).calls += 1
|
||||
self.send_response(type(self).status)
|
||||
self.end_headers()
|
||||
self.wfile.write(type(self).body)
|
||||
|
||||
def log_message(self, format, *args): pass
|
||||
return FakeHandler
|
||||
|
||||
|
||||
FakeHandler = make_handler()
|
||||
|
||||
|
||||
class DeliveryResponseTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.server = HTTPServer(('127.0.0.1', 0), FakeHandler)
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
||||
cls.thread.start()
|
||||
cls.url = 'http://127.0.0.1:%d/' % cls.server.server_port
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.shutdown()
|
||||
cls.server.server_close()
|
||||
|
||||
def setUp(self):
|
||||
FakeHandler.calls = 0
|
||||
FakeHandler.body = b'{"ok":true}'
|
||||
FakeHandler.status = 200
|
||||
|
||||
def request(self, method, expected):
|
||||
def send():
|
||||
req = Request(self.url, method=method)
|
||||
try:
|
||||
with urlopen(req, timeout=2) as response:
|
||||
return response.status, response.read(1024 * 1024 + 1)
|
||||
except HTTPError as e:
|
||||
return e.code, e.read(1024 * 1024 + 1)
|
||||
return dr.deliver_once(send, expected_status=expected)
|
||||
|
||||
def test_endpoint_specific_success_statuses(self):
|
||||
for method, status in [('GET', 200), ('POST', 201), ('PATCH', 201)]:
|
||||
with self.subTest(method=method, status=status):
|
||||
FakeHandler.status = status
|
||||
calls_before = FakeHandler.calls
|
||||
self.assertEqual(self.request(method, status), {'ok': True})
|
||||
self.assertEqual(FakeHandler.calls - calls_before, 1)
|
||||
|
||||
def test_rejects_invalid_expected_status_before_send(self):
|
||||
calls = []
|
||||
for invalid in (None, True, 0, 700, '201'):
|
||||
with self.subTest(invalid=invalid):
|
||||
with self.assertRaises(dr.InvalidExpectationError):
|
||||
dr.deliver_once(lambda: calls.append(1), expected_status=invalid)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_unexpected_success_status_uncertain_not_heuristic(self):
|
||||
FakeHandler.status = 202
|
||||
with self.assertRaises(dr.AcknowledgementUncertainError):
|
||||
self.request('POST', 201)
|
||||
self.assertEqual(FakeHandler.calls, 1)
|
||||
|
||||
def test_failure_status_cannot_be_configured_as_success(self):
|
||||
calls = []
|
||||
for status in (100, 199, 300, 400, 500, 599):
|
||||
with self.subTest(status=status), self.assertRaises(dr.InvalidExpectationError):
|
||||
dr.deliver_once(lambda: calls.append(1) or (status, b'{}'), expected_status=status)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_raised_auth_or_redirect_error_is_rejected_not_uncertain(self):
|
||||
for status in (300, 302, 304, 399, 401, 403):
|
||||
calls = []
|
||||
def send():
|
||||
calls.append(1)
|
||||
raise HTTPError('http://example.invalid/private', status, 'sensitive detail', {}, None)
|
||||
with self.subTest(status=status):
|
||||
with self.assertRaises(dr.RejectedResponseError) as caught:
|
||||
dr.deliver_once(send, expected_status=201)
|
||||
self.assertEqual(calls, [1])
|
||||
self.assertNotIn('sensitive detail', str(caught.exception))
|
||||
|
||||
def test_sender_explicit_rejection_is_not_reclassified_as_uncertain(self):
|
||||
def send():
|
||||
raise dr.RejectedResponseError('sensitive detail')
|
||||
with self.assertRaises(dr.RejectedResponseError) as caught:
|
||||
dr.deliver_once(send, expected_status=201)
|
||||
self.assertNotIn('sensitive detail', str(caught.exception))
|
||||
|
||||
def test_tls_certificate_rejection_is_not_reclassified_as_uncertain(self):
|
||||
import ssl
|
||||
from urllib.error import URLError
|
||||
for error in (ssl.SSLCertVerificationError('sensitive detail'),
|
||||
URLError(ssl.SSLCertVerificationError('sensitive detail'))):
|
||||
def send():
|
||||
raise error
|
||||
with self.subTest(error=type(error).__name__), self.assertRaises(dr.RejectedResponseError):
|
||||
dr.deliver_once(send, expected_status=201)
|
||||
|
||||
def test_auth_and_redirect_rejected(self):
|
||||
for status in (401, 403, 302):
|
||||
with self.subTest(status=status):
|
||||
FakeHandler.status = status
|
||||
with self.assertRaises(dr.RejectedResponseError):
|
||||
self.request('POST', 201)
|
||||
|
||||
def test_bad_json_shape_and_size_are_uncertain(self):
|
||||
for body in (b'{', b'[]', b'null', b'{"x":NaN}', b'"x"', b' ' * (1024 * 1024 + 1)):
|
||||
with self.subTest(body=body[:12]):
|
||||
FakeHandler.body = body
|
||||
FakeHandler.status = 201
|
||||
with self.assertRaises(dr.AcknowledgementUncertainError):
|
||||
self.request('POST', 201)
|
||||
|
||||
def test_transport_error_once_and_sanitized_uncertainty(self):
|
||||
calls = []
|
||||
def send():
|
||||
calls.append(1)
|
||||
raise OSError('secret token should not leak')
|
||||
with self.assertRaises(dr.AcknowledgementUncertainError) as caught:
|
||||
dr.deliver_once(send, expected_status=201)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertNotIn('secret token', str(caught.exception))
|
||||
self.assertIn('reconcil', str(caught.exception).lower())
|
||||
|
||||
def test_control_exceptions_propagate(self):
|
||||
for exc in (KeyboardInterrupt(), SystemExit()):
|
||||
with self.subTest(exc=type(exc).__name__):
|
||||
with self.assertRaises(type(exc)):
|
||||
dr.deliver_once(lambda: (_ for _ in ()).throw(exc), expected_status=200)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,152 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
sys.path.insert(0, str(ROOT / "harness" / "lib"))
|
||||
import impl_operation as op
|
||||
|
||||
|
||||
class ImplOperationTests(unittest.TestCase):
|
||||
def test_reentrant_preflight_cannot_authorize_duplicate_mutation(self):
|
||||
coordinator = op.Coordinator()
|
||||
intent = make_intent()
|
||||
approval = make_approval(intent)
|
||||
writes = []
|
||||
entered = []
|
||||
def preflight(value):
|
||||
if not entered:
|
||||
entered.append(True)
|
||||
coordinator.execute(intent, approval, provider)
|
||||
return {"fresh": True}
|
||||
provider = adapter(preflight=preflight, mutate=lambda value: writes.append("write"))
|
||||
coordinator.execute(intent, approval, provider)
|
||||
self.assertEqual(writes, ["write"])
|
||||
|
||||
def test_preflight_cannot_change_approved_payload_before_mutation(self):
|
||||
intent = make_intent()
|
||||
calls = []
|
||||
def preflight(i):
|
||||
i.payload["body"] = "unapproved"
|
||||
return {"fresh": True}
|
||||
c = op.Coordinator()
|
||||
result = c.execute(intent, make_approval(intent), adapter(preflight=preflight,
|
||||
mutate=lambda i: calls.append(i.payload["body"])), calls.append)
|
||||
self.assertEqual(calls[-1], "ok")
|
||||
self.assertEqual(result.status, "verified")
|
||||
|
||||
def test_attempt_callback_must_be_callable_before_preflight(self):
|
||||
calls = []
|
||||
result = op.Coordinator().execute(make_intent(), make_approval(),
|
||||
adapter(preflight=lambda i: calls.append("preflight") or {"fresh": True}), on_attempt=3)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertEqual(result.status, "partial")
|
||||
|
||||
def test_attempt_callback_failure_consumes_authority_without_write(self):
|
||||
calls = []
|
||||
c = op.Coordinator()
|
||||
a = adapter(mutate=lambda i: calls.append("write"))
|
||||
result = c.execute(make_intent(), make_approval(), a,
|
||||
lambda record: (_ for _ in ()).throw(RuntimeError("secret credential")))
|
||||
self.assertEqual(calls, [])
|
||||
self.assertNotIn("secret credential", result.reason)
|
||||
self.assertEqual(c.execute(make_intent(), make_approval(), a).status, "partial")
|
||||
|
||||
def test_malformed_canonical_target_rejected(self):
|
||||
for target in ("owner/..", "owner/.", "owner/repo\\x", "owner/repo\n"):
|
||||
with self.subTest(target=target):
|
||||
i = op.Intent("v1", "push", target, 1, "a"*40, {})
|
||||
self.assertRaises(ValueError, op.validate, i, op.Approval("v1", "0"*64, "approved"))
|
||||
|
||||
def test_unserializable_ack_does_not_lose_verified_result(self):
|
||||
result = op.Coordinator().execute(make_intent(), make_approval(), adapter(mutate=lambda i: object()))
|
||||
self.assertEqual(result.status, "verified")
|
||||
|
||||
def test_verify_exception_returns_partial_and_no_retry(self):
|
||||
calls = []
|
||||
c = op.Coordinator()
|
||||
a = adapter(mutate=lambda i: calls.append("write"), verify=lambda *args: (_ for _ in ()).throw(RuntimeError("private")))
|
||||
result = c.execute(make_intent(), make_approval(), a)
|
||||
self.assertEqual(result.status, "partial")
|
||||
self.assertNotIn("private", result.reason)
|
||||
c.execute(make_intent(), make_approval(), a)
|
||||
self.assertEqual(calls, ["write"])
|
||||
|
||||
def test_versioned_intent_and_approval_validate(self):
|
||||
intent = op.Intent("v1", "comment.patch", "owner/repo", 12, "a" * 40, {"body": "ok"})
|
||||
approval = op.Approval("v1", intent.digest, "approved")
|
||||
self.assertEqual(op.validate(intent, approval).digest, intent.digest)
|
||||
|
||||
def test_invalid_full_sha_rejected_before_callbacks(self):
|
||||
calls = []
|
||||
intent = op.Intent("v1", "push", "owner/repo", 1, "a" * 39, {})
|
||||
result = op.Coordinator().execute(intent, op.Approval("v1", "0" * 64, "approved"), adapter(), calls.append)
|
||||
self.assertEqual(result.status, "partial")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_stale_preflight_fails_without_mutation(self):
|
||||
calls = []
|
||||
result = op.Coordinator().execute(make_intent(), make_approval(), adapter(preflight=lambda i: {"fresh": False}), calls.append)
|
||||
self.assertEqual(result.status, "partial")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_unsupported_capability_fails_before_write(self):
|
||||
calls = []
|
||||
result = op.Coordinator().execute(make_intent(), make_approval(), adapter(capability=False), calls.append)
|
||||
self.assertEqual(result.status, "partial")
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_uncertain_ack_reconciles_without_second_write(self):
|
||||
calls = []
|
||||
a = adapter(mutate=lambda i: (_ for _ in ()).throw(op.UncertainAcknowledgement()), readback=lambda i: {"state": "applied"}, verify=lambda i, p, r: True)
|
||||
result = op.Coordinator().execute(make_intent(), make_approval(), a, calls.append)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(result.status, "verified")
|
||||
self.assertEqual(result.assurance["durable_at_most_once"], False)
|
||||
|
||||
def test_reuse_cannot_mutate_twice(self):
|
||||
calls = []
|
||||
c = op.Coordinator()
|
||||
a = adapter()
|
||||
c.execute(make_intent(), make_approval(), a, calls.append)
|
||||
again = c.execute(make_intent(), make_approval(), a, calls.append)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(again.status, "partial")
|
||||
|
||||
def test_verification_requires_trusted_adapter_callback(self):
|
||||
calls = []
|
||||
result = op.Coordinator().execute(make_intent(), make_approval(), adapter(verify=lambda *args: False), calls.append)
|
||||
self.assertEqual(result.status, "partial")
|
||||
self.assertNotEqual(result.status, "verified")
|
||||
|
||||
def test_correction_needs_separate_approval_unless_explicit_patch_exception(self):
|
||||
calls = []
|
||||
correction = make_intent(operation="correction")
|
||||
result = op.Coordinator().execute(correction, make_approval(correction), adapter(capability="ordinary_patch"), calls.append)
|
||||
self.assertEqual(result.status, "partial")
|
||||
self.assertEqual(calls, [])
|
||||
approval = make_approval(correction, correction=True, ordinary_patch=True)
|
||||
result = op.Coordinator().execute(correction, approval, adapter(capability="ordinary_patch"), calls.append)
|
||||
self.assertEqual(result.assurance["atomic_client_precondition"], False)
|
||||
self.assertIn("residual_race", result.assurance)
|
||||
|
||||
|
||||
def make_intent(operation="comment.patch"):
|
||||
return op.Intent("v1", operation, "owner/repo", 12, "a" * 40, {"body": "ok"})
|
||||
|
||||
|
||||
def make_approval(intent=None, correction=False, ordinary_patch=False):
|
||||
intent = intent or make_intent()
|
||||
return op.Approval("v1", intent.digest, "approved", correction=correction, ordinary_patch=ordinary_patch)
|
||||
|
||||
|
||||
def adapter(**kwargs):
|
||||
defaults = dict(preflight=lambda i: {"fresh": True}, capability=True,
|
||||
mutate=lambda i: {"written": True}, readback=lambda i: {"state": "applied"},
|
||||
verify=lambda i, p, r: True)
|
||||
defaults.update(kwargs)
|
||||
return op.Adapter(**defaults)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,150 @@
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
PATH = ROOT / "harness/agents/impl/skills/gitea-issue-implementation/scripts/implementation_delivery.py"
|
||||
|
||||
|
||||
def load(path, name):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class RecordingAdapter:
|
||||
def __init__(self, head, *, remote_head=None, pr=None, mutate=None):
|
||||
self.head = head
|
||||
self.remote_head = remote_head if remote_head is not None else head
|
||||
self.pr = pr if pr is not None else {"number": 17, "head": head, "state": "open"}
|
||||
self.mutate = mutate
|
||||
self.calls = []
|
||||
|
||||
def push(self, head):
|
||||
self.calls.append(("push", head))
|
||||
if self.mutate:
|
||||
self.mutate("push")
|
||||
|
||||
def read_remote_head(self):
|
||||
self.calls.append(("read_remote_head",))
|
||||
if self.mutate:
|
||||
self.mutate("read_remote_head")
|
||||
return self.remote_head
|
||||
|
||||
def create_or_reuse_pr(self, head):
|
||||
self.calls.append(("create_or_reuse_pr", head))
|
||||
if self.mutate:
|
||||
self.mutate("create_or_reuse_pr")
|
||||
return self.pr["number"]
|
||||
|
||||
def read_pr(self, number):
|
||||
self.calls.append(("read_pr", number))
|
||||
return self.pr
|
||||
|
||||
|
||||
class ImplementationDeliveryTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.delivery = load(PATH, "implementation_delivery_under_test")
|
||||
from harness.tests._support import delegation
|
||||
cls.evidence = delegation
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = TemporaryDirectory(dir="/opt/data/cache/scratch")
|
||||
self.ev = self.evidence.RealEvidence(Path(self.tmp.name))
|
||||
self.ev.set_stage("implementation-delivery")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_real_git_delivery_pushes_once_creates_once_and_returns_verified_readback(self):
|
||||
adapter = RecordingAdapter(self.ev.revision)
|
||||
result = self.delivery.deliver_implementation(
|
||||
adapter, head=self.ev.revision, declaration=self.ev.declaration()
|
||||
)
|
||||
self.assertEqual(result, {
|
||||
"remote_head": self.ev.revision,
|
||||
"pr": {"number": 17, "head": self.ev.revision, "state": "open"},
|
||||
})
|
||||
self.assertEqual(
|
||||
[call[0] for call in adapter.calls],
|
||||
["push", "read_remote_head", "create_or_reuse_pr", "read_pr"],
|
||||
)
|
||||
|
||||
def test_explicit_none_is_checked_and_exact_readback_is_required(self):
|
||||
adapter = RecordingAdapter(self.ev.revision)
|
||||
result = self.delivery.deliver_implementation(
|
||||
adapter, head=self.ev.revision, declaration={"mode": "none"}
|
||||
)
|
||||
self.assertEqual(result["remote_head"], self.ev.revision)
|
||||
self.assertEqual(result["pr"]["state"], "open")
|
||||
|
||||
def test_missing_or_malformed_declaration_has_no_effects(self):
|
||||
for declaration in (None, {"mode": "bad"}):
|
||||
adapter = RecordingAdapter(self.ev.revision)
|
||||
with self.subTest(declaration=declaration), self.assertRaises(self.delivery.AdoptionError):
|
||||
self.delivery.deliver_implementation(adapter, head=self.ev.revision, declaration=declaration)
|
||||
self.assertEqual(adapter.calls, [])
|
||||
|
||||
def test_remote_drift_prevents_pr_creation(self):
|
||||
adapter = RecordingAdapter(self.ev.revision, remote_head="0" * 40)
|
||||
with self.assertRaises(self.delivery.AdoptionError):
|
||||
self.delivery.deliver_implementation(adapter, head=self.ev.revision, declaration=self.ev.declaration())
|
||||
self.assertEqual([call[0] for call in adapter.calls], ["push", "read_remote_head"])
|
||||
|
||||
def test_evidence_mutation_during_remote_readback_prevents_pr_creation(self):
|
||||
def mutate(phase):
|
||||
if phase == "read_remote_head":
|
||||
self.ev.bundle.joinpath("src/a.py").write_text("VALUE = 2\n")
|
||||
|
||||
adapter = RecordingAdapter(self.ev.revision, mutate=mutate)
|
||||
with self.assertRaises(self.delivery.AdoptionError):
|
||||
self.delivery.deliver_implementation(adapter, head=self.ev.revision, declaration=self.ev.declaration())
|
||||
self.assertEqual([call[0] for call in adapter.calls], ["push", "read_remote_head"])
|
||||
|
||||
def test_missing_adapter_capability_prevents_push(self):
|
||||
class Incomplete:
|
||||
def read_remote_head(self):
|
||||
return self.ev.revision
|
||||
|
||||
with self.assertRaises(self.delivery.AdoptionError):
|
||||
self.delivery.deliver_implementation(Incomplete(), head=self.ev.revision, declaration={"mode": "none"})
|
||||
|
||||
def test_pr_readback_mismatch_does_not_complete(self):
|
||||
for pr in (
|
||||
{"number": 17, "head": "0" * 40, "state": "open"},
|
||||
{"number": 17, "head": self.ev.revision, "state": "closed"},
|
||||
{"number": True, "head": self.ev.revision, "state": "open"},
|
||||
):
|
||||
adapter = RecordingAdapter(self.ev.revision, pr=pr)
|
||||
with self.subTest(pr=pr), self.assertRaises(self.delivery.AdoptionError):
|
||||
self.delivery.deliver_implementation(adapter, head=self.ev.revision, declaration=self.ev.declaration())
|
||||
|
||||
def test_workflow_can_load_from_external_cwd_without_pythonpath(self):
|
||||
import os
|
||||
import subprocess
|
||||
code = "import importlib.util,json; s=importlib.util.spec_from_file_location('delivery'," + repr(str(PATH)) + "); m=importlib.util.module_from_spec(s); s.loader.exec_module(m); head='a'*40; A=type('A',(),{'push':lambda self,h:None,'read_remote_head':lambda self:head,'create_or_reuse_pr':lambda self,h:17,'read_pr':lambda self,n:{'number':n,'head':head,'state':'open'}}); print(json.dumps(m.deliver_implementation(A(),head=head,declaration={'mode':'none'})))"
|
||||
env = os.environ.copy(); env.pop('PYTHONPATH', None)
|
||||
proc = subprocess.run([sys.executable, '-c', code], cwd=self.tmp.name, env=env,
|
||||
capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(proc.returncode, 0, 'canonical delivery must load independently of cwd')
|
||||
self.assertIn('remote_head', proc.stdout)
|
||||
|
||||
def test_uncertain_write_is_not_retried(self):
|
||||
class FailingAdapter(RecordingAdapter):
|
||||
def create_or_reuse_pr(self, head):
|
||||
self.calls.append(("create_or_reuse_pr", head))
|
||||
raise RuntimeError("transport uncertain")
|
||||
|
||||
adapter = FailingAdapter(self.ev.revision)
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.delivery.deliver_implementation(adapter, head=self.ev.revision, declaration=self.ev.declaration())
|
||||
self.assertEqual([call[0] for call in adapter.calls], ["push", "read_remote_head", "create_or_reuse_pr"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Integration regression tests for coordinator-backed initial delivery."""
|
||||
import importlib.util
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from harness.tests._support.paths import ROOT
|
||||
SCRIPT = ROOT / "harness/agents/impl/skills/gitea-issue-implementation/scripts/implementation_delivery.py"
|
||||
sys.path.insert(0, str(ROOT))
|
||||
spec = importlib.util.spec_from_file_location("implementation_delivery", SCRIPT)
|
||||
delivery = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(delivery)
|
||||
from harness.lib.impl_operation import Approval, Coordinator, Intent
|
||||
|
||||
HEAD = "a" * 40
|
||||
|
||||
class Adapter:
|
||||
def __init__(self):
|
||||
self.events=[]; self.number=7; self.fail_pr=False; self.uncertain=False; self.received=[]
|
||||
def push_operation(self, intent):
|
||||
self.received.append(intent)
|
||||
return self.push(intent.payload['head'])
|
||||
def create_or_reuse_pr_operation(self, intent):
|
||||
self.received.append(intent)
|
||||
return self.create_or_reuse_pr(intent.payload['head'])
|
||||
def push(self, head): self.events.append("push"); return True
|
||||
def read_remote_head(self): self.events.append("read-head"); return HEAD
|
||||
def create_or_reuse_pr(self, head):
|
||||
self.events.append("create-pr")
|
||||
if self.fail_pr: raise RuntimeError("failed")
|
||||
if self.uncertain: raise RuntimeError("uncertain ack")
|
||||
return self.number
|
||||
def read_pr(self, number): self.events.append("read-pr"); return {"number":number,"head":HEAD,"state":"open"}
|
||||
def preflight_operation(self, intent): self.events.append("preflight:"+intent.operation); return {"fresh":True,"scope":intent.target}
|
||||
def verify_operation(self, intent, before, observed): self.events.append("verify:"+intent.operation); return True
|
||||
def find_intended_pr(self, head): self.events.append("find-pr"); return None if self.fail_pr else {"number":self.number}
|
||||
|
||||
def make_intents():
|
||||
push=Intent("v1","push","org/repo",42,HEAD,{"head":HEAD})
|
||||
pr=Intent("v1","pr.create","org/repo",42,HEAD,{"head":HEAD,"body":"approved body","base":"main"})
|
||||
return push, Approval("v1",push.digest,"approved"), pr, Approval("v1",pr.digest,"approved")
|
||||
|
||||
class IntegrationTests(unittest.TestCase):
|
||||
def invoke(self, adapter, intents=None, approvals=None, recorder=None):
|
||||
p,pa,r,ra=make_intents(); intents=intents or (p,r); approvals=approvals or (pa,ra)
|
||||
return delivery.deliver_approved_implementation(adapter,head=HEAD,declaration={"mode":"none"},push_intent=intents[0],push_approval=approvals[0],pr_intent=intents[1],pr_approval=approvals[1],coordinator=Coordinator(),on_attempt=recorder or (lambda entry: None))
|
||||
def test_full_composed_delivery(self):
|
||||
a=Adapter(); result=self.invoke(a)
|
||||
self.assertEqual(result["status"],"verified"); self.assertEqual(result["artifact"]["pr"]["number"],7)
|
||||
self.assertEqual(a.events.count("push"),1); self.assertEqual(a.events.count("create-pr"),1)
|
||||
def test_uncertain_pr_acknowledgement_reconciles_without_second_write(self):
|
||||
a = Adapter()
|
||||
a.uncertain = True
|
||||
result = self.invoke(a)
|
||||
self.assertEqual(result["status"], "verified")
|
||||
self.assertEqual(a.events.count("create-pr"), 1)
|
||||
self.assertEqual(a.events.count("find-pr"), 1)
|
||||
|
||||
def test_adoption_rejection_never_promotes_existing_remote_state(self):
|
||||
a = Adapter()
|
||||
old = delivery._delivery_adoption.require_adoption
|
||||
delivery._delivery_adoption.require_adoption = lambda *args, **kwargs: (_ for _ in ()).throw(ValueError("rejected"))
|
||||
try:
|
||||
result = self.invoke(a)
|
||||
finally:
|
||||
delivery._delivery_adoption.require_adoption = old
|
||||
self.assertEqual(result["status"], "partial")
|
||||
self.assertNotIn("push", a.events)
|
||||
self.assertNotIn("create-pr", a.events)
|
||||
|
||||
def test_approved_payload_head_mismatch_rejected_before_any_write(self):
|
||||
a = Adapter()
|
||||
push, push_approval, pr, pr_approval = make_intents()
|
||||
for operation in ('push', 'pr.create'):
|
||||
candidate = Intent('v1', operation, 'org/repo', 42, HEAD, {'head':'f'*40,'body':'approved'})
|
||||
candidate_approval = Approval('v1',candidate.digest,'approved')
|
||||
intents = (candidate,pr) if operation == 'push' else (push,candidate)
|
||||
approvals = (candidate_approval,pr_approval) if operation == 'push' else (push_approval,candidate_approval)
|
||||
with self.subTest(operation=operation),self.assertRaises(ValueError):
|
||||
self.invoke(a,intents=intents,approvals=approvals)
|
||||
self.assertNotIn('push',a.events)
|
||||
self.assertNotIn('create-pr',a.events)
|
||||
|
||||
def test_mutation_callbacks_receive_entire_approved_intent(self):
|
||||
a = Adapter()
|
||||
push, push_approval, pr, pr_approval = make_intents()
|
||||
self.assertEqual(self.invoke(a)['status'],'verified')
|
||||
self.assertEqual([item.as_dict() for item in a.received],[push.as_dict(),pr.as_dict()])
|
||||
|
||||
def test_new_adapter_contract_does_not_require_legacy_mutators(self):
|
||||
a = Adapter()
|
||||
a.push = None
|
||||
a.create_or_reuse_pr = None
|
||||
a.push_operation = lambda intent: a.events.append('push')
|
||||
a.create_or_reuse_pr_operation = lambda intent: a.events.append('create-pr') or 7
|
||||
self.assertEqual(self.invoke(a)['status'],'verified')
|
||||
self.assertEqual(a.events.count('push'),1)
|
||||
self.assertEqual(a.events.count('create-pr'),1)
|
||||
|
||||
def test_missing_approval_no_writes(self):
|
||||
a=Adapter(); p,pa,r,ra=make_intents(); ra=Approval("v1","0"*64,"approved")
|
||||
with self.assertRaises(ValueError): self.invoke(a,approvals=(pa,ra))
|
||||
self.assertNotIn("push",a.events)
|
||||
def test_preflight_failure_no_mutation(self):
|
||||
a=Adapter(); a.preflight_operation=lambda intent:{"fresh":False}
|
||||
result=self.invoke(a); self.assertEqual(result["status"],"partial"); self.assertNotIn("push",a.events)
|
||||
def test_pr_failure_preserves_verified_push(self):
|
||||
a=Adapter(); a.fail_pr=True; result=self.invoke(a)
|
||||
self.assertEqual(result["status"],"partial"); self.assertEqual(result["push"].status,"verified"); self.assertIsNone(result["artifact"])
|
||||
def test_mutations_each_require_fresh_adoption(self):
|
||||
a=Adapter(); calls=[]; old=delivery._delivery_adoption.require_adoption
|
||||
delivery._delivery_adoption.require_adoption=lambda *args,**kwargs:calls.append(kwargs["expected_stage"])
|
||||
try: self.invoke(a)
|
||||
finally: delivery._delivery_adoption.require_adoption=old
|
||||
self.assertEqual(len(calls),4)
|
||||
|
||||
if __name__ == "__main__": unittest.main()
|
||||
Reference in New Issue
Block a user