Add shared managed lock broker

This commit is contained in:
2026-10-07 02:23:39 +00:00
parent 18676070b2
commit 470c7fb09b
16 changed files with 1583 additions and 120 deletions
View File
+263
View File
@@ -0,0 +1,263 @@
import fcntl
import os
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from harness.lib.managed_lock import (
LockOrderError,
LockTimeoutError,
LockValidationError,
ManagedLock,
ManagedLockSet,
probe_free,
validate_held_flock,
)
class ManagedLockTests(unittest.TestCase):
def test_lock_is_created_securely_and_held_for_context(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "repository.lock"
with ManagedLock(path) as lock:
self.assertIsNotNone(lock.fd)
st = path.stat()
self.assertEqual(stat.S_IMODE(st.st_mode), 0o600)
self.assertEqual(st.st_nlink, 1)
self.assertEqual(st.st_uid, os.getuid())
self.assertTrue(validate_held_flock(lock.fd, path))
self.assertTrue(probe_free(path))
def test_lock_timeout_is_bounded_and_does_not_steal_lock(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "issue.lock"
with ManagedLock(path):
start = time.monotonic()
with self.assertRaises(LockTimeoutError):
with ManagedLock(path, timeout=0.05, poll_interval=0.005):
pass
self.assertLess(time.monotonic() - start, 1)
self.assertTrue(probe_free(path))
def test_existing_unsafe_lock_file_is_rejected_without_repair(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "bad.lock"
path.write_text("x")
os.chmod(path, 0o644)
with self.assertRaises(LockValidationError):
ManagedLock(path).acquire()
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o644)
self.assertEqual(path.read_text(), "x")
def test_literal_dotdot_component_is_rejected_before_normalization(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "a" / ".." / "b.lock"
with self.assertRaises(LockValidationError):
ManagedLock(path)
def test_symlink_lock_is_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
target = root / "target"
target.write_text("do not touch")
path = root / "link.lock"
path.symlink_to(target)
with self.assertRaises(LockValidationError):
ManagedLock(path).acquire()
self.assertEqual(target.read_text(), "do not touch")
def test_symlink_to_safe_lock_is_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
target = root / "safe-target.lock"
target.touch(mode=0o600)
path = root / "safe-link.lock"
path.symlink_to(target)
with self.assertRaises(LockValidationError):
ManagedLock(path)
def test_lock_path_through_symlinked_parent_is_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
target = root / "real"
target.mkdir()
parent = root / "linked"
parent.symlink_to(target, target_is_directory=True)
with self.assertRaises(LockValidationError):
ManagedLock(parent / "issue.lock").acquire()
self.assertFalse((target / "issue.lock").exists())
def test_ancestor_swap_after_parent_open_cannot_redirect_lock(self):
"""The final open stays relative to the already pinned parent inode."""
with tempfile.TemporaryDirectory() as d:
root = Path(d)
parent = root / "locks"
parent.mkdir()
diverted = root / "diverted"
diverted.mkdir()
module = __import__("harness.lib.managed_lock", fromlist=["_open_parent"])
original_open_parent = module._open_parent
def swap_after_open(path, *, create_parent):
fd, name = original_open_parent(path, create_parent=create_parent)
parent.rename(root / "original-locks")
parent.symlink_to(diverted, target_is_directory=True)
return fd, name
lock = ManagedLock(parent / "issue.lock")
with patch("harness.lib.managed_lock._open_parent", side_effect=swap_after_open):
with self.assertRaises(LockValidationError):
lock.acquire()
self.assertTrue((root / "original-locks" / "issue.lock").exists())
self.assertFalse((diverted / "issue.lock").exists())
self.assertFalse(lock.held)
def test_hardlink_lock_is_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
path = root / "lock"
alias = root / "alias"
path.touch(mode=0o600)
os.link(path, alias)
with self.assertRaises(LockValidationError):
ManagedLock(path).acquire()
def test_for_issue_builds_canonical_paths(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d).absolute()
issue, repository = ManagedLockSet.paths_for_issue(root, "alice", "ledger", 7)
self.assertEqual(issue, root / "repositories/alice/ledger/locks/issue-7.lock")
self.assertEqual(repository, root / "repositories/alice/ledger/locks/repository.lock")
locks = ManagedLockSet.for_issue(root, "alice", "ledger", 7)
self.assertEqual(locks.issue.path, issue)
self.assertEqual(locks.repository.path, repository)
def test_for_issue_rejects_unsafe_identity_issue_and_root(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d).absolute()
for owner, repo in [("../alice", "ledger"), ("alice/x", "ledger"), ("alice", ".."), ("alice", "")]:
with self.subTest(owner=owner, repo=repo), self.assertRaises(ValueError):
ManagedLockSet.for_issue(root, owner, repo, 1)
for issue in (0, -1, True, "1"):
with self.subTest(issue=issue), self.assertRaises(ValueError):
ManagedLockSet.for_issue(root, "alice", "ledger", issue)
with self.assertRaises(ValueError):
ManagedLockSet.for_issue(Path("relative"), "alice", "ledger", 1)
with self.assertRaises(ValueError):
ManagedLockSet.for_issue(root / ".." / root.name, "alice", "ledger", 1)
def test_managed_lock_set_holds_issue_then_repository_scope_only(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
issue, repo = root / "issue", root / "repo"
events = []
locks = ManagedLockSet(issue, repo, event_sink=events.append)
with locks:
self.assertTrue(validate_held_flock(locks.issue.fd, issue))
self.assertFalse(locks.repository.held)
with locks.repository_scope():
self.assertTrue(validate_held_flock(locks.repository.fd, repo))
self.assertFalse(locks.repository.held)
self.assertEqual(events, ["acquire:issue", "acquire:repository", "release:repository", "release:issue"])
def test_release_issue_rejects_while_repository_held(self):
with tempfile.TemporaryDirectory() as d:
locks = ManagedLockSet(Path(d) / "issue", Path(d) / "repo")
with locks:
with locks.repository_scope():
with self.assertRaises(LockOrderError):
locks.release_issue()
locks.release_issue()
self.assertFalse(locks.issue.held)
def test_context_exception_cleans_up_issue_and_repository(self):
with tempfile.TemporaryDirectory() as d:
locks = ManagedLockSet(Path(d) / "issue", Path(d) / "repo")
with self.assertRaises(RuntimeError):
with locks:
with locks.repository_scope():
raise RuntimeError("boom")
self.assertFalse(locks.issue.held)
self.assertFalse(locks.repository.held)
self.assertTrue(probe_free(locks.issue.path))
self.assertTrue(probe_free(locks.repository.path))
def test_repository_scope_requires_issue_and_cleans_up_acquire_exception(self):
with tempfile.TemporaryDirectory() as d:
locks = ManagedLockSet(Path(d) / "issue", Path(d) / "repo")
with self.assertRaises(LockOrderError):
with locks.repository_scope():
pass
with ManagedLock(locks.repository.path):
with locks:
with self.assertRaises(LockTimeoutError):
with locks.repository_scope():
pass
self.assertTrue(locks.issue.held)
locks.release_issue()
def test_repository_scope_event_exception_releases_repository_and_issue_context(self):
with tempfile.TemporaryDirectory() as d:
def fail_on_repository(event):
if event == "acquire:repository":
raise RuntimeError("event failure")
locks = ManagedLockSet(Path(d) / "issue", Path(d) / "repo", event_sink=fail_on_repository)
locks.issue.acquire()
with self.assertRaises(RuntimeError):
with locks.repository_scope():
pass
self.assertTrue(locks.issue.held)
self.assertFalse(locks.repository.held)
locks.release_issue()
self.assertTrue(probe_free(locks.issue.path))
self.assertTrue(probe_free(locks.repository.path))
def test_write_metadata_replaces_bytes_exactly_on_repeated_writes(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "metadata.lock"
with ManagedLock(path) as lock:
lock.write_metadata({"long": "x" * 100})
lock.write_metadata({"x": 1})
self.assertEqual(path.read_bytes(), b'{"x":1}\n')
def test_metadata_requires_held_lock(self):
with tempfile.TemporaryDirectory() as d:
lock = ManagedLock(Path(d) / "metadata.lock")
with self.assertRaises(RuntimeError):
lock.write_metadata({"state": "held"})
def test_probe_free_does_not_delete_or_modify_lock(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "probe.lock"
with ManagedLock(path):
pass
before = path.stat()
self.assertTrue(probe_free(path))
after = path.stat()
self.assertEqual((before.st_ino, before.st_size, before.st_mode), (after.st_ino, after.st_size, after.st_mode))
def test_other_process_cannot_acquire_while_context_alive(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "process.lock"
script = """
import sys
sys.path.insert(0, sys.argv[2])
from harness.lib.managed_lock import ManagedLock, LockTimeoutError
try:
with ManagedLock(sys.argv[1], timeout=0.03, poll_interval=0.005):
print('acquired')
except LockTimeoutError:
print('blocked')
"""
with ManagedLock(path):
result = subprocess.run([sys.executable, "-c", script, str(path), str(Path.cwd())], text=True, capture_output=True, check=True)
self.assertEqual(result.stdout.strip(), "blocked")
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,261 @@
import json
import os
import secrets
import stat
import subprocess
import sys
import tempfile
import time
import unittest
from argparse import Namespace
from pathlib import Path
from unittest.mock import patch
from harness.lib.managed_lock import ManagedLock, probe_free
from harness.lib import managed_lock_broker as broker
BROKER = Path(__file__).parents[2] / "lib" / "managed_lock_broker.py"
def wait_ready(proc, timeout=3):
line = proc.stdout.readline().decode()
if not line:
raise AssertionError(proc.stderr.read().decode())
return json.loads(line)
class ManagedLockBrokerTests(unittest.TestCase):
def start(self, root, owner="alice", repo="ledger", issue=7, ttl=5, role="gitea-impl-agent"):
locks = root / "repositories" / owner / repo / "locks"
locks.mkdir(parents=True)
os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"
runtime.mkdir()
os.chmod(runtime, 0o700)
handle = runtime / "handle.json"
socket = runtime / "broker.sock"
cmd = [sys.executable, str(BROKER), "serve", "--root", str(root), "--owner", owner,
"--repository", repo, "--issue", str(issue), "--role", role, "--run-id", "run-1",
"--handle", str(handle), "--socket", str(socket), "--ttl-seconds", str(ttl),
"--acquire-timeout", "0.3"]
proc = subprocess.Popen(cmd, cwd=Path.cwd(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ready = wait_ready(proc)
self.assertEqual(ready["status"], "held")
self.assertNotIn("token", json.dumps(ready))
return proc, handle, socket
def request(self, handle, action, ttl=None, token=None):
data = json.loads(handle.read_text())
if token is not None:
data["token"] = token
import socket
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as s:
s.connect(data["socket"])
payload = {"action": action, "token": data["token"] if token is None else token}
if ttl is not None:
payload["ttl_seconds"] = ttl
s.sendall((json.dumps(payload) + "\n").encode())
with s.makefile("rb") as response_file:
return json.loads(response_file.readline())
def test_cross_process_contention_and_authenticated_release(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
proc, handle, _ = self.start(root)
try:
self.assertFalse(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
self.assertEqual(self.request(handle, "release")["status"], "released")
proc.wait(timeout=2)
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
self.assertNotIn("token", json.loads(handle.read_text()))
finally:
if proc.poll() is None:
proc.kill(); proc.wait()
proc.stdout.close(); proc.stderr.close()
def test_probe_cli_fails_while_held_and_passes_after_release(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
proc, handle, _ = self.start(root)
command = [sys.executable, str(BROKER), "probe", "--root", str(root),
"--owner", "alice", "--repository", "ledger", "--issue", "7"]
try:
held = subprocess.run(command, capture_output=True, text=True)
self.assertEqual(held.returncode, 1)
self.assertFalse(json.loads(held.stdout)["free"])
self.assertEqual(self.request(handle, "release")["status"], "released")
proc.wait(timeout=2)
free = subprocess.run(command, capture_output=True, text=True)
self.assertEqual(free.returncode, 0)
self.assertTrue(json.loads(free.stdout)["free"])
finally:
if proc.poll() is None:
proc.kill(); proc.wait()
proc.stdout.close(); proc.stderr.close()
def test_issue_probe_passes_when_another_workflow_holds_repository(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
proc, handle, _ = self.start(root)
base = [sys.executable, str(BROKER), "probe", "--root", str(root),
"--owner", "alice", "--repository", "ledger", "--issue", "7"]
repository = root / "repositories/alice/ledger/locks/repository.lock"
try:
self.assertEqual(self.request(handle, "release")["status"], "released")
proc.wait(timeout=2)
with ManagedLock(repository):
issue = subprocess.run([*base, "--resource", "issue"], capture_output=True, text=True)
all_resources = subprocess.run([*base, "--resource", "all"], capture_output=True, text=True)
self.assertEqual(issue.returncode, 0)
self.assertTrue(json.loads(issue.stdout)["issue_free"])
self.assertEqual(all_resources.returncode, 1)
self.assertFalse(json.loads(all_resources.stdout)["repository_free"])
finally:
if proc.poll() is None:
proc.kill(); proc.wait()
proc.stdout.close(); proc.stderr.close()
def test_new_handle_mode_is_exact_under_restrictive_umask(self):
with tempfile.TemporaryDirectory() as d:
path = Path(d) / "handle.json"
old = os.umask(0o777)
try:
broker._write_handle(path, {"token": "a" * 64, "socket": "/tmp/x"}, exclusive=True)
finally:
os.umask(old)
self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600)
def test_repository_scope_release_keeps_issue(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root)
try:
self.assertEqual(self.request(handle, "acquire_repository")["status"], "repository-held")
self.assertFalse(probe_free(root / "repositories/alice/ledger/locks/repository.lock"))
self.assertEqual(self.request(handle, "release_repository")["status"], "held")
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/repository.lock"))
self.assertFalse(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
finally:
self.request(handle, "release"); proc.wait(timeout=2); proc.stdout.close(); proc.stderr.close()
def test_ttl_expiry_updates_handle_and_releases(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root, ttl=1)
proc.wait(timeout=4)
state = json.loads(handle.read_text())
self.assertEqual(state["status"], "expired")
self.assertNotIn("token", state)
self.assertTrue(probe_free(root / "repositories/alice/ledger/locks/issue-7.lock"))
proc.stdout.close(); proc.stderr.close()
def test_expired_lease_rejects_renew_and_release_before_action(self):
for action in ("renew", "release"):
with self.subTest(action=action), tempfile.TemporaryDirectory() as d:
root = Path(d)
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
args = Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="expiry-test",
handle=str(runtime / "handle.json"), socket=str(runtime / "broker.sock"),
ttl_seconds=5.0, acquire_timeout=0.0)
instance = broker.Broker(args)
instance.set.issue.acquire()
broker._write_handle(instance.handle, instance.state, exclusive=True)
instance.handle_created = True; instance.started = True
instance.deadline = time.monotonic() - 1
request: dict[str, object] = {"action": action, "token": instance.token}
if action == "renew": request["ttl_seconds"] = 30
self.assertEqual(instance.handle_request(request), {"error": "expired"})
self.assertTrue(probe_free(locks / "issue-7.lock"))
terminal = json.loads(instance.handle.read_text())
self.assertEqual(terminal["status"], "expired")
self.assertNotIn("token", terminal)
def test_startup_failure_releases_issue_and_terminalizes_partial_handle(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
handle = runtime / "handle.json"
args = Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="startup-failure",
handle=str(handle), socket=str(runtime / "broker.sock"),
ttl_seconds=5.0, acquire_timeout=0.0)
instance = broker.Broker(args)
real_write = broker._write_handle
def partial_then_fail(path, state, *, exclusive=False, on_created=None):
real_write(path, state, exclusive=exclusive, on_created=on_created)
raise OSError("synthetic startup failure")
with patch.object(broker, "_write_handle", side_effect=partial_then_fail):
with self.assertRaises(OSError):
instance.run()
self.assertTrue(probe_free(locks / "issue-7.lock"))
terminal = json.loads(handle.read_text())
self.assertIn(terminal["status"], ("released", "expired"))
self.assertNotIn("token", terminal)
def test_startup_existing_handle_is_never_claimed_or_modified(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
locks = root / "repositories/alice/ledger/locks"
locks.mkdir(parents=True); os.chmod(locks, 0o700)
(locks / "repository.lock").touch(mode=0o600)
runtime = root / "runtime"; runtime.mkdir(mode=0o700)
handle = runtime / "handle.json"
original = b'{"foreign":"do-not-touch"}\n'
handle.write_bytes(original); handle.chmod(0o600)
args = Namespace(root=str(root), owner="alice", repository="ledger", issue=7,
role="gitea-impl-agent", run_id="existing-handle",
handle=str(handle), socket=str(runtime / "broker.sock"),
ttl_seconds=5.0, acquire_timeout=0.0)
instance = broker.Broker(args)
with self.assertRaises(FileExistsError):
instance.run()
self.assertEqual(handle.read_bytes(), original)
self.assertTrue(probe_free(locks / "issue-7.lock"))
def test_cli_client_compatibility_and_nonprinting_function(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root)
try:
result = broker.request_handle(handle, "status")
self.assertEqual(result["status"], "held")
self.assertNotIn("token", json.dumps(result))
cli = subprocess.run([sys.executable, str(BROKER), "request", "--handle", str(handle), "--action", "status"],
capture_output=True, text=True)
self.assertEqual(cli.returncode, 0)
self.assertEqual(json.loads(cli.stdout)["status"], "held")
finally:
broker.request_handle(handle, "release"); proc.wait(timeout=2); proc.stdout.close(); proc.stderr.close()
def test_token_not_in_ready_or_errors_and_bad_handle_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d); proc, handle, _ = self.start(root)
try:
raw = handle.read_text()
self.assertIn("token", raw)
bad = root / "bad"; bad.symlink_to(handle)
result = subprocess.run([sys.executable, str(BROKER), "request", "--handle", str(bad), "--action", "status"], capture_output=True, text=True)
self.assertNotEqual(result.returncode, 0)
self.assertNotIn(json.loads(raw)["token"], result.stderr)
self.assertEqual(self.request(handle, "status", token=secrets.token_hex(32))["error"], "unauthorized")
finally:
self.request(handle, "release"); proc.wait(timeout=2); proc.stdout.close(); proc.stderr.close()
def test_unsafe_handle_and_identifiers_rejected(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
for role in ("bad", "gitea-impl-agent/evil"):
p = subprocess.run([sys.executable, str(BROKER), "serve", "--root", str(root), "--owner", "a", "--repository", "r", "--issue", "1", "--role", role, "--run-id", "x", "--handle", str(root/"h"), "--socket", str(root/"s"), "--ttl-seconds", "1"], capture_output=True, text=True)
self.assertNotEqual(p.returncode, 0)
self.assertNotIn("token", p.stderr.lower())
if __name__ == "__main__":
unittest.main()