Files
gitea-agent/harness/tests/common/test_managed_lock_broker.py
T

466 lines
26 KiB
Python

import json
import os
import secrets
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from argparse import Namespace
from pathlib import Path
from unittest.mock import patch
from harness.lib.managed_lock import ManagedLock, probe_free
from harness.lib import managed_lock_broker as broker
BROKER = Path(__file__).parents[2] / "lib" / "managed_lock_broker.py"
def wait_ready(proc, timeout=3):
line = proc.stdout.readline().decode()
if not line:
raise AssertionError(proc.stderr.read().decode())
return json.loads(line)
class ManagedLockBrokerTests(unittest.TestCase):
def start(self, root, owner="alice", repo="ledger", issue=7, ttl=5, role="gitea-impl-agent", mask=-1):
locks = root / "repositories" / owner / repo / "locks"
locks.mkdir(parents=True)
os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
(locks / f"issue-{issue}.lock").touch(mode=0o600)
runtime = root / "runtime"
runtime.mkdir()
os.chmod(runtime, 0o700)
handle = runtime / "handle.json"
socket = runtime / "broker.sock"
cmd = [sys.executable, str(BROKER), "serve", "--root", str(root), "--owner", owner,
"--repository", repo, "--issue", str(issue), "--role", role, "--run-id", "run-1",
"--handle", str(handle), "--socket", str(socket), "--ttl-seconds", str(ttl),
"--acquire-timeout", "0.3"]
proc = subprocess.Popen(cmd, cwd=Path.cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, umask=mask)
ready = wait_ready(proc)
self.assertEqual(ready["status"], "held")
self.assertNotIn("token", json.dumps(ready))
return proc, handle, socket
def request(self, handle, action, ttl=None, token=None):
data = json.loads(handle.read_text())
if token is not None:
data["token"] = token
import socket
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
s.connect(data["socket"])
payload = {"action": action, "token": data["token"] if token is None else token}
if ttl is not None:
payload["ttl_seconds"] = ttl
s.sendall((json.dumps(payload) + "\n").encode())
with s.makefile("rb") as response_file:
return json.loads(response_file.readline())
def test_cross_process_contention_and_authenticated_release(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
proc, handle, _ = self.start(root)
try:
self.assertFalse(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
self.assertEqual(self.request(handle, "release")["status"], "released")
proc.wait(timeout=2)
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
self.assertNotIn("token", json.loads(handle.read_text()))
finally:
if proc.poll() is None:
proc.kill(); proc.wait()
proc.stdout.close(); proc.stderr.close()
def test_probe_cli_fails_while_held_and_passes_after_release(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
proc, handle, _ = self.start(root)
command = [sys.executable, str(BROKER), "probe", "--root", str(root),
"--owner", "alice", "--repository", "ledger", "--issue", "7"]
try:
held = subprocess.run(command, capture_output=True, text=True)
self.assertEqual(held.returncode, 1)
self.assertFalse(json.loads(held.stdout)["free"])
self.assertEqual(self.request(handle, "release")["status"], "released")
proc.wait(timeout=2)
free = subprocess.run(command, capture_output=True, text=True)
self.assertEqual(free.returncode, 0)
self.assertTrue(json.loads(free.stdout)["free"])
finally:
if proc.poll() is None:
proc.kill(); proc.wait()
proc.stdout.close(); proc.stderr.close()
def test_issue_probe_passes_when_another_workflow_holds_repository(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
proc, handle, _ = self.start(root)
base = [sys.executable, str(BROKER), "probe", "--root", str(root),
"--owner", "alice", "--repository", "ledger", "--issue", "7"]
repository = root / "repositories/alice/ledger/locks/repository.lock"
try:
self.assertEqual(self.request(handle, "release")["status"], "released")
proc.wait(timeout=2)
with ManagedLock(repository):
issue = subprocess.run([*base, "--resource", "issue"], capture_output=True, text=True)
all_resources = subprocess.run([*base, "--resource", "all"], capture_output=True, text=True)
self.assertEqual(issue.returncode, 0)
self.assertTrue(json.loads(issue.stdout)["issue_free"])
self.assertEqual(all_resources.returncode, 1)
self.assertFalse(json.loads(all_resources.stdout)["repository_free"])
finally:
if proc.poll() is None:
proc.kill(); proc.wait()
proc.stdout.close(); proc.stderr.close()
def test_new_handle_mode_is_exact_under_restrictive_umask(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "handle.json"
old = os.umask(0o777)
try:
broker._write_handle(path, {"token": "a" * 64, "socket": "/tmp/x"}, exclusive=True)
finally:
os.umask(old)
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600)
def test_socket_mode_and_cleanup_under_umasks(self):
for mask in (0o022, 0o077, 0o777):
with self.subTest(mask=mask), tempfile.TemporaryDirectory() as d:
proc, handle, sock = self.start(Path(d), mask=mask)
try:
self.assertEqual(stat.S_IMODE(sock.lstat().st_mode), 0o600)
self.assertTrue(stat.S_ISSOCK(sock.lstat().st_mode))
self.assertEqual(broker.request_handle(handle, "status")["status"], "held")
self.assertEqual(broker.request_handle(handle, "release")["status"], "released")
self.assertEqual(proc.wait(timeout=3), 0)
self.assertFalse(sock.exists())
self.assertTrue(probe_free(Path(d) / "repositories/alice/ledger/locks/issue-7.lock"))
finally:
if proc.poll() is None:
proc.terminate(); proc.wait(timeout=3)
proc.stdout.close(); proc.stderr.close()
def make_bound_instance(self, root):
import socket
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); locks.chmod(0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
instance = broker.Broker(Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="cleanup-test", handle=str(runtime / "handle.json"),
socket=str(runtime / "broker.sock"), ttl_seconds=5.0, acquire_timeout=0.0))
instance.set.issue.acquire()
broker._write_handle(instance.handle, instance.state, exclusive=True,
on_created=instance._record_handle_identity)
instance.handle_created = True; instance.started = True
instance.socket_parent_fd = broker._open_directory(runtime)
instance.server = socket.socket(socket.AF_UNIX)
instance.server.bind(str(instance.sock_path))
st = instance.sock_path.lstat()
instance.socket_identity = (st.st_dev, st.st_ino)
return instance
def test_release_reports_socket_failure_separately_and_stops(self):
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d))
with patch.object(broker.os, "unlink", side_effect=PermissionError("injected")):
result = instance.handle_request({"action": "release", "token": instance.token})
self.assertEqual(result["error"], "cleanup-incomplete")
self.assertTrue(result["cleanup"]["lease_released"])
self.assertFalse(result["cleanup"]["socket_removed"])
self.assertTrue(instance.stopping)
self.assertTrue(probe_free(instance.set.issue.path))
self.assertNotIn("token", json.loads(instance.handle.read_text()))
self.assertIsNone(instance.socket_parent_fd)
self.assertIsNone(instance.server)
def test_cleanup_preserves_replaced_path_and_reports_incomplete(self):
for kind in ("file", "symlink"):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d))
instance.sock_path.unlink()
target = Path(d) / "target"; target.write_text("foreign")
if kind == "file": instance.sock_path.write_text("replacement")
else: instance.sock_path.symlink_to(target)
self.assertFalse(instance.cleanup("released"))
self.assertTrue(instance.sock_path.is_symlink() if kind == "symlink" else instance.sock_path.is_file())
self.assertEqual(target.read_text(), "foreign")
self.assertTrue(instance.cleanup_result["lease_released"])
self.assertIn("socket-identity-changed", instance.cleanup_result["errors"])
self.assertFalse(instance.cleanup("released")) # Do not erase a partial result.
def test_socket_removed_after_expiry_and_sigterm(self):
for action in ("expiry", "sigterm"):
with self.subTest(action=action), tempfile.TemporaryDirectory() as d:
proc, handle, sock = self.start(Path(d), ttl=1 if action == "expiry" else 5)
try:
if action == "sigterm": proc.terminate()
self.assertEqual(proc.wait(timeout=4), 0)
self.assertFalse(sock.exists())
self.assertTrue(probe_free(Path(d) / "repositories/alice/ledger/locks/issue-7.lock"))
self.assertNotIn("token", json.loads(handle.read_text()))
finally:
if proc.poll() is None: proc.kill(); proc.wait(timeout=3)
proc.stdout.close(); proc.stderr.close()
def test_partial_cleanup_response_delivered_before_nonzero_exit(self):
import socket
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, sock_path = self.start(root)
target = root / "replacement-target"; target.write_text("untouched")
try:
data = json.loads(handle.read_text())
with socket.socket(socket.AF_UNIX) as client:
client.settimeout(3); client.connect(str(sock_path))
sock_path.unlink(); sock_path.symlink_to(target)
client.sendall((json.dumps({"action": "release", "token": data["token"]}) + "\n").encode())
with client.makefile("rb") as stream:
result = json.loads(stream.readline())
self.assertEqual(result["error"], "cleanup-incomplete")
self.assertTrue(result["cleanup"]["lease_released"])
self.assertEqual(proc.wait(timeout=3), 2)
self.assertTrue(sock_path.is_symlink())
self.assertEqual(target.read_text(), "untouched")
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
terminal = json.loads(handle.read_text())
self.assertNotIn("token", terminal)
self.assertFalse(terminal["cleanup"]["socket_removed"])
self.assertNotIn(data["token"], proc.stderr.read().decode())
finally:
if proc.poll() is None: proc.kill(); proc.wait(timeout=3)
proc.stdout.close(); proc.stderr.close()
def test_socket_permission_failure_has_no_ready_and_cleans_owned_socket(self):
import contextlib
import io
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d))
instance.cleanup("released")
args = Namespace(root=str(instance.root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="startup-permission", handle=str(instance.handle.parent / "new.handle"),
socket=str(instance.sock_path.parent / "new.sock"), ttl_seconds=5.0, acquire_timeout=0.0)
fresh = broker.Broker(args)
out = io.StringIO()
with patch.object(broker.os, "chmod", side_effect=PermissionError("injected")), contextlib.redirect_stdout(out):
with self.assertRaises(PermissionError): fresh.run()
self.assertEqual(out.getvalue(), "")
self.assertFalse(fresh.sock_path.exists())
self.assertTrue(probe_free(fresh.set.issue.path))
self.assertNotIn("token", json.loads(fresh.handle.read_text()))
def test_cleanup_respects_lock_order_after_repository_error(self):
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d))
instance.set.repository.acquire()
with patch.object(instance.set.repository, "release", side_effect=OSError("injected")):
self.assertFalse(instance.cleanup("released"))
self.assertFalse(probe_free(instance.set.issue.path)) # Repository still held; preserve lock order.
self.assertFalse(instance.cleanup_result["lease_released"])
self.assertTrue(instance.cleanup_result["socket_removed"])
self.assertIn("repository-release", instance.cleanup_result["errors"])
instance.cleanup("released") # Retry held FD, but keep previous failure visible.
self.assertTrue(probe_free(instance.set.repository.path))
self.assertNotIn("token", json.loads(instance.handle.read_text()))
def test_startup_handle_replacement_is_preserved(self):
import contextlib
import io
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d)); instance.cleanup("released")
fresh = broker.Broker(Namespace(root=str(instance.root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="handle-swap", handle=str(instance.handle.parent / "swap.handle"),
socket=str(instance.sock_path.parent / "swap.sock"), ttl_seconds=5.0, acquire_timeout=0.0))
original_write = broker._write_handle
foreign = b'{"foreign":"preserve"}\n'
def replace_after_create(path, state, **kwargs):
original_write(path, state, **kwargs)
path.rename(path.with_suffix(".original")) # Keep original inode alive.
path.write_bytes(foreign); path.chmod(0o600)
with patch.object(broker, "_write_handle", side_effect=replace_after_create), contextlib.redirect_stdout(io.StringIO()):
with self.assertRaises((OSError, ValueError)): fresh.run()
self.assertEqual(fresh.handle.read_bytes(), foreign)
self.assertTrue(probe_free(fresh.set.issue.path))
self.assertIn("handle-save", fresh.cleanup_result["errors"])
def test_partial_cleanup_durable_status_is_incomplete(self):
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d))
with patch.object(broker.os, "unlink", side_effect=PermissionError("injected")):
instance.cleanup("released")
state = json.loads(instance.handle.read_text())
self.assertEqual(state["status"], "cleanup-incomplete")
self.assertTrue(state["cleanup"]["lease_released"])
self.assertFalse(state["cleanup"]["socket_removed"])
self.assertNotIn("token", state)
def test_socket_replacement_during_permission_setup_is_preserved(self):
import contextlib
import io
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d)); instance.cleanup("released")
fresh = broker.Broker(Namespace(root=str(instance.root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="socket-swap", handle=str(instance.handle.parent / "swap.handle"),
socket=str(instance.sock_path.parent / "swap.sock"), ttl_seconds=5.0, acquire_timeout=0.0))
real_chmod = os.chmod
def replace_after_chmod(path, mode, **kwargs):
real_chmod(path, mode, **kwargs)
fresh.sock_path.unlink()
fresh.sock_path.write_text("foreign socket replacement")
out = io.StringIO()
with patch.object(broker.os, "chmod", side_effect=replace_after_chmod), contextlib.redirect_stdout(out):
with self.assertRaises(ValueError): fresh.run()
self.assertEqual(out.getvalue(), "")
self.assertEqual(fresh.sock_path.read_text(), "foreign socket replacement")
self.assertTrue(probe_free(fresh.set.issue.path))
self.assertEqual(json.loads(fresh.handle.read_text())["status"], "cleanup-incomplete")
def test_persist_rejects_replaced_handle_after_ready(self):
with tempfile.TemporaryDirectory() as d:
instance = self.make_bound_instance(Path(d))
instance.handle.rename(instance.handle.with_suffix(".original"))
instance.handle.write_text("foreign"); instance.handle.chmod(0o600)
with self.assertRaises(ValueError): instance.persist("held")
self.assertFalse(instance.cleanup("released"))
self.assertEqual(instance.handle.read_text(), "foreign")
self.assertTrue(probe_free(instance.set.issue.path))
def test_repository_scope_release_keeps_issue(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root)
try:
self.assertEqual(self.request(handle, "acquire_repository")["status"], "repository-held")
self.assertFalse(probe_free(root / "repositories/alice/ledger/locks/repository.lock"))
self.assertEqual(self.request(handle, "release_repository")["status"], "held")
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/repository.lock"))
self.assertFalse(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
finally:
self.request(handle, "release"); proc.wait(timeout=2); proc.stdout.close(); proc.stderr.close()
def test_ttl_expiry_updates_handle_and_releases(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root, ttl=1)
proc.wait(timeout=4)
state = json.loads(handle.read_text())
self.assertEqual(state["status"], "expired")
self.assertNotIn("token", state)
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
proc.stdout.close(); proc.stderr.close()
def test_expired_lease_rejects_renew_and_release_before_action(self):
for action in ("renew", "release"):
with self.subTest(action=action), tempfile.TemporaryDirectory() as d:
root = Path(d)
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
args = Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="expiry-test",
handle=str(runtime / "handle.json"), socket=str(runtime / "broker.sock"),
ttl_seconds=5.0, acquire_timeout=0.0)
instance = broker.Broker(args)
instance.set.issue.acquire()
broker._write_handle(instance.handle, instance.state, exclusive=True)
instance.handle_created = True; instance.started = True
instance.deadline = time.monotonic() - 1
request: dict[str, object] = {"action": action, "token": instance.token}
if action == "renew": request["ttl_seconds"] = 30
self.assertEqual(instance.handle_request(request), {"error": "expired"})
self.assertTrue(probe_free(locks / "issue-7.lock"))
terminal = json.loads(instance.handle.read_text())
self.assertEqual(terminal["status"], "expired")
self.assertNotIn("token", terminal)
def test_startup_failure_releases_issue_and_terminalizes_partial_handle(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
handle = runtime / "handle.json"
args = Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="startup-failure",
handle=str(handle), socket=str(runtime / "broker.sock"),
ttl_seconds=5.0, acquire_timeout=0.0)
instance = broker.Broker(args)
real_write = broker._write_handle
def partial_then_fail(path, state, *, exclusive=False, on_created=None):
real_write(path, state, exclusive=exclusive, on_created=on_created)
raise OSError("synthetic startup failure")
with patch.object(broker, "_write_handle", side_effect=partial_then_fail):
with self.assertRaises(OSError):
instance.run()
self.assertTrue(probe_free(locks / "issue-7.lock"))
terminal = json.loads(handle.read_text())
self.assertIn(terminal["status"], ("released", "expired"))
self.assertNotIn("token", terminal)
def test_startup_existing_handle_is_never_claimed_or_modified(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
handle = runtime / "handle.json"
original = b'{"foreign":"do-not-touch"}\n'
handle.write_bytes(original); handle.chmod(0o600)
args = Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="existing-handle",
handle=str(handle), socket=str(runtime / "broker.sock"),
ttl_seconds=5.0, acquire_timeout=0.0)
instance = broker.Broker(args)
with self.assertRaises(FileExistsError):
instance.run()
self.assertEqual(handle.read_bytes(), original)
self.assertTrue(probe_free(locks / "issue-7.lock"))
def test_cli_client_compatibility_and_nonprinting_function(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root)
try:
result = broker.request_handle(handle, "status")
self.assertEqual(result["status"], "held")
self.assertEqual(result["root"], str(root))
self.assertEqual(broker.read_handle_metadata(handle)["root"], str(root))
self.assertNotIn("token", json.dumps(result))
cli = subprocess.run([sys.executable, str(BROKER), "request", "--handle", str(handle), "--action", "status"],
capture_output=True, text=True)
self.assertEqual(cli.returncode, 0)
self.assertEqual(json.loads(cli.stdout)["status"], "held")
finally:
broker.request_handle(handle, "release"); proc.wait(timeout=2); proc.stdout.close(); proc.stderr.close()
def test_token_not_in_ready_or_errors_and_bad_handle_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root)
try:
raw = handle.read_text()
self.assertIn("token", raw)
bad = root / "bad"; bad.symlink_to(handle)
result = subprocess.run([sys.executable, str(BROKER), "request", "--handle", str(bad), "--action", "status"], capture_output=True, text=True)
self.assertNotEqual(result.returncode, 0)
self.assertNotIn(json.loads(raw)["token"], result.stderr)
self.assertEqual(self.request(handle, "status", token=secrets.token_hex(32))["error"], "unauthorized")
finally:
self.request(handle, "release"); proc.wait(timeout=2); proc.stdout.close(); proc.stderr.close()
def test_unsafe_handle_and_identifiers_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
for role in ("bad", "gitea-impl-agent/evil"):
p = subprocess.run([sys.executable, str(BROKER), "serve", "--root", str(root), "--owner", "a", "--repository", "r", "--issue", "1", "--role", role, "--run-id", "x", "--handle", str(root/"h"), "--socket", str(root/"s"), "--ttl-seconds", "1"], capture_output=True, text=True)
self.assertNotEqual(p.returncode, 0)
self.assertNotIn("token", p.stderr.lower())
if __name__ == "__main__":
unittest.main()