From ba83fae581a289cbe4548d1226ba6122ed5a4ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:12:57 +0100 Subject: [PATCH 01/10] fix(skills/bounding-heavy-commands): give back slots while a command waits for memory The wrapper took its slots, and then it waited until the free memory held its budget. While it waited, it kept the slots but had no scope. So `--check` showed no line for it, and a smaller command that fit in the free memory waited behind slots that nothing used. Now the wrapper gives back its slots while the free memory does not hold its budget. When the memory looks free again, it takes slots through the line and reads the memory again under the memory lock before it starts. So two waiters that see the same free memory cannot both start on it. The wait has the same no-memory exit 75 when no other bounded command runs. While it waits for memory, the wrapper holds a wait file locked in the lock directory. The file names the budget, the slots, the start time, the directory, and the command. It holds no process ID. `--check` prints one `check waiting ... for=memory` line for each wait file, also without the hard cap, and removes a wait file that no process holds. Fixes #80. Co-Authored-By: Claude Opus 5.5 --- .../scripts/bounded_run.py | 148 +++++++++++++-- .../scripts/tests/test_bounded_run.py | 169 +++++++++++++++++- 2 files changed, 293 insertions(+), 24 deletions(-) diff --git a/skills/bounding-heavy-commands/scripts/bounded_run.py b/skills/bounding-heavy-commands/scripts/bounded_run.py index 5efaf7e..cbd8023 100755 --- a/skills/bounding-heavy-commands/scripts/bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/bounded_run.py @@ -47,11 +47,13 @@ --check runs no command and holds no lock when it returns. Its exit code is 0. It prints the state of the queue at that moment, one held line for each -running bounded command where the hard cap is available, and one line for -each --memory, in the order given: +running bounded command where the hard cap is available, one waiting line +for each bounded command that waits for memory, and one line for each +--memory, in the order given: bounded-run: check slots=N slot=M free-slots=N|- free=M|- unused=M|- cpus=N cpus-used=N|- cpus-headroom=N cpus-free=N|- line=free|busy fits=M|- bounded-run: check held unit=NAME budget=M|- used=M|- cpus=N|- age=Ns|- dir=PATH|- command=TEXT|- + bounded-run: check waiting unit=NAME for=memory budget=M slots=N age=Ns dir=PATH|- command=TEXT|- bounded-run: check budget=M slots=N headroom=M starts=yes|no A '-' stands for a value that the check cannot read or count here. cpus is @@ -62,7 +64,9 @@ cpus-headroom and cpus-used, and at least 0. starts=yes means that a run of that budget would start now. It is not a reservation. fits is the largest budget whose memory, with its own headroom, is free now, in steps of 256M, -or 0M when none is; it does not count slots or the line. +or 0M when none is; it does not count slots or the line. A command that +waits for memory holds none of the slots of its waiting line, so a smaller +command whose budget fits can start before it. A normal call sets none of the environment variables below. A person can set the tuning values in the profile of the shell, so that every session @@ -93,6 +97,7 @@ import fcntl import hashlib +import json import math import os import random @@ -512,6 +517,76 @@ def running_commands(directory: str) -> Optional[int]: return running +def wait_file_name(unit: str) -> str: + """Returns the name of the file that a bounded command holds locked while it waits for memory.""" + return "wait-%s.lock" % unit + + +def write_wait_file( + directory: str, unit: str, budget_mib: int, slots: int, working: str, command: Sequence[str], + now: Callable[[], float] = time.time, +) -> int: + """Writes the wait file of the unit and holds it locked. Returns its descriptor for remove_wait_file. + + The file names the budget, the slots, the directory, and the command, for memory_waiters to read. It holds no process ID. + """ + path = os.path.join(directory, wait_file_name(unit)) + partial = path + ".new" + text = json.dumps({"budget": budget_mib, "slots": slots, "since": now(), "dir": working, "command": list(command)}) + descriptor = _open_lock_file(partial) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX) + os.write(descriptor, text.encode("utf-8")) + # A reader takes a wait file that no process holds as one that a killed wrapper left, so the file gets its name only once it is locked and full. + os.rename(partial, path) + except OSError as error: + os.close(descriptor) + raise WrapperError("cannot write %s: %s" % (path, error.strerror if error.strerror else error)) + return descriptor + + +def remove_wait_file(directory: str, unit: str, descriptor: int) -> None: + try: + os.unlink(os.path.join(directory, wait_file_name(unit))) + except OSError: + pass + os.close(descriptor) + + +def memory_waiters(directory: str, now: Callable[[], float] = time.time) -> List[str]: + """Returns one text for each bounded command that waits for memory, as the fields of a check line. + + Returns nothing when the directory cannot be read. A wait file that no process holds is left over from a wrapper + that was killed, and is removed. + """ + try: + names = sorted(os.listdir(directory)) + except OSError: + return [] + waiters = [] + for name in names: + if not (name.startswith("wait-") and name.endswith(".lock")): + continue + path = os.path.join(directory, name) + descriptor = _try_lock(path) + if descriptor is not None: + try: + os.unlink(path) + except OSError: + pass + os.close(descriptor) + continue + try: + fields = json.loads(_read_text(path) or "") + waiters.append("unit=%s for=memory budget=%dM slots=%d age=%ds dir=%s command=%s" % ( + name[len("wait-"):-len(".lock")], fields["budget"], fields["slots"], max(0, int(now() - fields["since"])), + _one_line(fields["dir"]) or "-", _one_line(" ".join(fields["command"]))[:HELD_COMMAND_CHARACTERS] or "-")) + except (ValueError, KeyError, TypeError): + # The waiter removed its file between the lock and the read. + continue + return waiters + + def _lock_error(path: str, error: OSError) -> WrapperError: error_text = error.strerror if error.strerror else str(error) return WrapperError("cannot lock %s: %s. The lock directory must be on a filesystem that supports file locks." % (path, error_text)) @@ -703,14 +778,21 @@ def wait_for_memory( lock: Callable[[], None] = lambda: None, unlock: Callable[[], None] = lambda: None, clock: Callable[[], float] = time.monotonic, + hold: Callable[[], None] = lambda: None, + release: Callable[[], None] = lambda: None, ) -> None: """Returns when the free memory, less the unused budgets of running commands and the headroom, holds the budget. + The caller holds the slots of the command at the call. While the memory is not free, the call gives them back + with release, so that a smaller command can start with them. When the memory is free again, it takes them with + hold, with lock released, and reads the memory again before it returns. hold can wait in the line for minutes. + There is no time limit while another bounded command runs, so the call can block for as long as the machine is full. Raises NoMemory, with lock released, when none runs, because then nothing in the queue frees memory. read_running gives the number of bounded commands that run, or None where it is not known: the call waits. Returns at once when the free memory cannot be read. read_outstanding gives None where the unused budgets cannot be counted. - The call returns with lock held: the caller calls unlock once the new command counts in read_outstanding. + The call returns with lock held and the slots held: the caller calls unlock once the new command counts in read_outstanding. + NoMemory leaves the slots as they are. """ def read() -> Tuple[Optional[int], Optional[int]]: # A running command that grows or shrinks during the reads moves both values. The larger of the two unused @@ -728,31 +810,34 @@ def state(available: int, outstanding: Optional[int]) -> str: started = clock() next_line: Optional[float] = None + held = True while True: lock() available, outstanding = read() - if available is None: - log("cannot read the free memory; starting") - return - if memory_fits(budget_mib, available, outstanding, headroom_mib): - log(state(available, outstanding) + "; starting") - return - if read_running() == 0: + fits = available is None or memory_fits(budget_mib, available, outstanding, headroom_mib) + if not fits and read_running() == 0: # A command ends without the memory lock, so the last one can free its memory after the reads above. available, outstanding = read() - if available is None: - log("cannot read the free memory; starting") - return - if memory_fits(budget_mib, available, outstanding, headroom_mib): - log(state(available, outstanding) + "; starting") + fits = available is None or memory_fits(budget_mib, available, outstanding, headroom_mib) + if not fits: + unlock() + raise NoMemory(available, outstanding) + if fits: + if held: + log("cannot read the free memory; starting" if available is None else state(available, outstanding) + "; starting") return unlock() - raise NoMemory(available, outstanding) + hold() + held = True + continue waited = clock() - started if next_line is None or waited >= next_line: log(state(available, outstanding) + ("; waiting" if next_line is None else "; still waiting after %ds" % waited)) next_line = waited + interval_seconds unlock() + if held: + release() + held = False sleep(poll_seconds) @@ -1588,8 +1673,27 @@ def read_running() -> Optional[int]: return max(running or 0, found) return running + try: + working = os.getcwd() + except OSError: + working = "-" + reservations: List[Reservation] = [] + waiting: List[int] = [] + + def take_slots() -> None: + while waiting: + remove_wait_file(directory, unit, waiting.pop()) + reservations.append(reserve(directory, count, needed, exclusive_files, settings.poll_seconds)) + + def give_back_slots() -> None: + while reservations: + reservations.pop().release() + log("giving back %d slot(s) while waiting for free memory" % needed) + if not waiting: + waiting.append(write_wait_file(directory, unit, wait_budget, needed, working, options.command)) + reported: Optional[int] = None - reservation = reserve(directory, count, needed, exclusive_files, settings.poll_seconds) + take_slots() try: try: wait_for_memory( @@ -1597,6 +1701,7 @@ def read_running() -> Optional[int]: headroom_mib=headroom_mib(settings.total_mib, wait_budget, most), read_available=read_available, read_outstanding=read_outstanding, read_running=read_running, lock=lock_memory, unlock=unlock_memory, + hold=take_slots, release=give_back_slots, ) except NoMemory as stop: log("no-memory budget=%dM free=%dM headroom=%dM fits=%dM" % ( @@ -1617,7 +1722,10 @@ def read_running() -> Optional[int]: os.close(run_lock.pop()) if capped: reported = read_peak_file(peak_file) - reservation.release() + while waiting: + remove_wait_file(directory, unit, waiting.pop()) + while reservations: + reservations.pop().release() exact = reported is not None peak = max(reported or tracked or 0, rusage_peak_mib(maxrss)) @@ -1686,6 +1794,8 @@ def mib(value: Optional[int]) -> str: "-" if available is None else "%dM" % largest_fit_mib(available, outstanding, settings.total_mib, most))) for held in held_commands(scopes, ()) if scopes is not None else []: log("check held " + held) + for waiter in memory_waiters(directory): + log("check waiting " + waiter) for budget in budgets: needed = slots_needed(budget, settings.slot_mib, count) headroom = headroom_mib(settings.total_mib, min(budget, most), most) diff --git a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py index 38b2ee7..e330192 100644 --- a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py @@ -724,6 +724,70 @@ def test_a_run_file_that_no_process_holds_is_removed_and_not_counted(self): self.assertFalse(os.path.exists(path)) +class MemoryWaiterTest(unittest.TestCase): + def setUp(self): + self.base = tempfile.TemporaryDirectory() + self.addCleanup(self.base.cleanup) + self.directory = self.base.name + + def mark(self, unit, budget=6144, slots=3, working="/work/project", command=("make", "test"), since=1000.0): + descriptor = bounded_run.write_wait_file(self.directory, unit, budget, slots, working, list(command), now=lambda: since) + self.addCleanup(lambda: os.close(descriptor) if self.is_open(descriptor) else None) + return descriptor + + @staticmethod + def is_open(descriptor): + try: + os.fstat(descriptor) + except OSError: + return False + return True + + def waiters(self, now=1042.0): + return bounded_run.memory_waiters(self.directory, now=lambda: now) + + def test_names_the_budget_the_slots_the_age_the_directory_and_the_command_of_each_waiter(self): + self.mark("bounded-run-5-000005") + self.mark("bounded-run-6-000006", budget=2048, slots=1, working="/other", command=("tool",), since=1040.0) + self.assertEqual(self.waiters(), [ + "unit=bounded-run-5-000005 for=memory budget=6144M slots=3 age=42s dir=/work/project command=make test", + "unit=bounded-run-6-000006 for=memory budget=2048M slots=1 age=2s dir=/other command=tool", + ]) + + def test_no_waiter_after_the_wait_file_is_removed(self): + descriptor = self.mark("bounded-run-5-000005") + bounded_run.remove_wait_file(self.directory, "bounded-run-5-000005", descriptor) + self.assertEqual(self.waiters(), []) + self.assertEqual(os.listdir(self.directory), []) + + def test_a_wait_file_that_no_process_holds_is_removed_and_not_named(self): + os.close(self.mark("bounded-run-5-000005")) + self.assertEqual(self.waiters(), []) + self.assertEqual(os.listdir(self.directory), []) + + def test_the_wait_file_holds_no_process_id(self): + self.mark("bounded-run-5-000005") + text = Path(self.directory, bounded_run.wait_file_name("bounded-run-5-000005")).read_text() + self.assertNotIn(str(os.getpid()), text) + + def test_a_directory_that_cannot_be_read_gives_no_waiter(self): + self.assertEqual(bounded_run.memory_waiters(os.path.join(self.directory, "missing")), []) + + def test_a_directory_and_a_command_with_line_breaks_stay_on_one_line(self): + self.mark("bounded-run-5-000005", working="/a\nb", command=("sh", "-c", "a\nb")) + self.assertEqual(self.waiters(), ["unit=bounded-run-5-000005 for=memory budget=6144M slots=3 age=42s dir=/a b command=sh -c a b"]) + + def test_a_long_command_is_cut(self): + self.mark("bounded-run-5-000005", command=("x" * 500,)) + self.assertEqual(len(self.waiters()[0].split(" command=")[1]), bounded_run.HELD_COMMAND_CHARACTERS) + + def test_slot_and_run_files_are_not_waiters(self): + for name in ("slot-000.lock", bounded_run.run_file_name("bounded-run-1-000001")): + descriptor = bounded_run._try_lock(os.path.join(self.directory, name)) + self.addCleanup(os.close, descriptor) + self.assertEqual(self.waiters(), []) + + class MemoryReadTest(unittest.TestCase): def test_reads_the_available_memory_of_meminfo(self): text = "MemTotal: 32000000 kB\nMemFree: 1000000 kB\nMemAvailable: 8388608 kB\n" @@ -873,8 +937,9 @@ def sleep(seconds): sleeps.append(seconds) now[0] += seconds * oversleep + # The last value stays, for the reading after the slots are taken again. bounded_run.wait_for_memory( - 4096, limit, 5.0, headroom_mib=headroom, read_available=lambda: readings.pop(0), + 4096, limit, 5.0, headroom_mib=headroom, read_available=lambda: readings.pop(0) if len(readings) > 1 else readings[0], read_outstanding=lambda: outstanding, sleep=sleep, clock=lambda: now[0], ) return sleeps @@ -924,8 +989,59 @@ def read(): bounded_run.wait_for_memory( 4096, 300, 5.0, read_available=read, sleep=lambda seconds: events.append("sleep"), lock=lambda: events.append("lock"), unlock=lambda: events.append("unlock"), + hold=lambda: readings.append(5000), ) - self.assertEqual(events, ["lock", "read", "unlock", "sleep", "lock", "read"]) + self.assertEqual(events, ["lock", "read", "unlock", "sleep", "lock", "read", "unlock", "lock", "read"]) + + def slot_events(self, values, running=1): + events = [] + readings = list(values) + + def read(): + events.append("read") + return readings.pop(0) + + bounded_run.wait_for_memory( + 4096, 300, 5.0, read_available=read, read_running=lambda: running, sleep=lambda seconds: events.append("sleep"), + lock=lambda: events.append("lock"), unlock=lambda: events.append("unlock"), + hold=lambda: events.append("hold"), release=lambda: events.append("release"), + ) + return events + + def test_keeps_the_slots_when_the_memory_is_free_at_once(self): + self.assertEqual(self.slot_events([8000]), ["lock", "read"]) + + def test_gives_back_the_slots_while_it_waits_and_takes_them_again_before_it_reads_the_memory_that_starts_it(self): + self.assertEqual(self.slot_events([1000, 1000, 8000, 8000]), [ + "lock", "read", "unlock", "release", "sleep", + "lock", "read", "unlock", "sleep", + "lock", "read", "unlock", "hold", + "lock", "read", + ]) + + def test_gives_back_the_slots_again_when_the_memory_is_taken_while_it_waits_for_them(self): + self.assertEqual(self.slot_events([1000, 8000, 1000, 8000, 8000]), [ + "lock", "read", "unlock", "release", "sleep", + "lock", "read", "unlock", "hold", + "lock", "read", "unlock", "release", "sleep", + "lock", "read", "unlock", "hold", + "lock", "read", + ]) + + def test_says_starting_only_when_it_holds_the_slots(self): + self.slot_events([1000, 8000, 8000]) + starting = [line for line in self.lines() if line.endswith("; starting")] + self.assertEqual(len(starting), 1, self.lines()) + + def test_stops_without_the_slots_when_the_memory_does_not_come_free_and_no_bounded_command_runs_any_more(self): + running = [1, 0] + events = [] + with self.assertRaises(bounded_run.NoMemory): + bounded_run.wait_for_memory( + 4096, 300, 5.0, read_available=lambda: 1000, read_running=lambda: running.pop(0), + sleep=lambda seconds: None, hold=lambda: events.append("hold"), release=lambda: events.append("release"), + ) + self.assertEqual(events, ["release"]) def test_reads_the_unused_budgets_before_and_after_the_free_memory(self): events = [] @@ -946,8 +1062,8 @@ def read_outstanding(): def test_counts_the_larger_of_the_unused_budgets_read_before_and_after_the_free_memory(self): for first, second in ((1000, 3000), (3000, 1000)): with self.subTest(first=first, second=second): - unused = [first, second, 0, 0] - readings = [4000, 8000] + unused = [first, second, 0, 0, 0, 0] + readings = [4000, 8000, 8000] sleeps = [] bounded_run.wait_for_memory( 2048, 300, 5.0, read_available=lambda: readings.pop(0), read_outstanding=lambda: unused.pop(0), @@ -980,7 +1096,7 @@ def test_stops_when_the_memory_is_not_free_and_no_bounded_command_runs(self): self.assertEqual(raised.exception.available, 1000) def test_waits_while_another_bounded_command_runs(self): - readings = [1000, 5000] + readings = [1000, 5000, 5000] sleeps = [] bounded_run.wait_for_memory(4096, 300, 5.0, read_available=lambda: readings.pop(0), read_running=lambda: 1, sleep=sleeps.append) @@ -2631,6 +2747,13 @@ def free_slots(*arguments, **keywords): self.assertEqual(events[-1], "slots") self.assertIn("probe", events) + def test_names_each_command_that_waits_for_memory_also_without_a_cap(self): + descriptor = bounded_run.write_wait_file(self.directory, "bounded-run-5-000005", 6144, 3, "/w", ["tool"], now=time.time) + self.addCleanup(os.close, descriptor) + lines = self.check(["2G"]) + self.assertRegex(lines[1], r"^check waiting unit=bounded-run-5-000005 for=memory budget=6144M slots=3 age=\d+s dir=/w command=tool$") + self.assertTrue(lines[2].startswith("check budget=2048M "), lines) + def test_no_held_line_without_a_cap(self): self.hold("slot-000.lock") self.assertFalse(any(line.startswith("check held") for line in self.check(["2G"]))) @@ -3003,6 +3126,42 @@ def test_a_waiting_command_holds_no_slot_and_is_not_overtaken(self): self.assertLess(order.index(["end", "holder"]), order.index(["start", "large"])) self.assertLess(order.index(["start", "large"]), order.index(["start", "small"])) + def memory_waiter(self): + """Starts a holder, then a large command whose budget does not fit in the free memory, with five slots of 2 GiB. + + Each wrapper reads 6000 MiB of free memory and keeps a headroom of 1228 MiB. The large command needs 3 slots, + and the 4 GiB command needs 2 of the 4 slots that the holder leaves. Returns the release file, the holder, and the + large command, once the large command waits for memory. + """ + self.environ.update({"BOUNDED_RUN_TOTAL_MIB": "12288", "BOUNDED_RUN_FREE_MIB": "6000"}) + release = os.path.join(self.base.name, "release") + holder = self.worker("holder", "2G", hold="300", release=release) # runs until the test writes the release file + self.wait_for_event("start", "holder") + large = self.worker("large", "6G", hold="0") # does not start: its budget never fits in the free memory + self.wait_for_stderr(large, "; waiting") + return release, holder, large + + def test_a_command_that_waits_for_memory_does_not_stop_a_smaller_command_that_fits(self): + release, holder, large = self.memory_waiter() + small = self.worker("small", "4G", hold="0") # ends at once; the test waits for it to end + self.assertEqual(small.wait(timeout=60), 0) + self.assertIsNone(holder.poll()) + Path(release).write_text("") + self.assertEqual(holder.wait(timeout=60), 0) + self.assertEqual(large.wait(timeout=60), bounded_run.EXIT_NO_MEMORY) + self.assertNotIn(["start", "large"], [line.split()[:2] for line in self.read_events()]) + self.assertEqual([name for name in os.listdir(self.environ["BOUNDED_RUN_LOCK_DIR"]) if name.startswith("wait-")], []) + + def test_a_check_names_the_command_that_waits_for_memory(self): + release, holder, large = self.memory_waiter() + done = subprocess.run([sys.executable, WRAPPER, "--check"], env=self.environ, capture_output=True, text=True, timeout=60) + self.assertRegex(done.stderr, r"\nbounded-run: check waiting unit=bounded-run-%d-[0-9a-f]{6} for=memory budget=6144M slots=3 age=\d+s dir=%s command=%s -c import os, sys, time def event" % ( + large.pid, re.escape(os.getcwd()), re.escape(sys.executable))) + self.assertIn(" free-slots=4 ", done.stderr) + Path(release).write_text("") + holder.wait(timeout=60) + large.wait(timeout=60) + def test_a_hard_kill_of_the_wrapper_frees_the_slots_at_once(self): holder = self.worker("holder", "4G", hold="300") # runs until the kill below ends it self.wait_for_event("start", "holder") From 5ac91770db848de236fb130f5401b4d92a766210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:14:54 +0100 Subject: [PATCH 02/10] docs(skills/bounding-heavy-commands): explain the check line of a command that waits for memory The wrapper now gives back its slots while it waits for free memory, and `--check` prints a `check waiting ... for=memory` line for each such command. The skill did not say how to read that line. The skill now gives the line format. It says that a waiting command holds no slot and has no place in the line, and that the line means the programs outside the queue hold the memory, not the slots. The report after a long wait, and the rule that a dispatcher gives its subagents, now name the waiting lines too. The lock files now carry no claim that they are empty, because a wait file names its command. In a baseline, agents that read a check with two waiting lines took them as an order in the line ("head of the line", "behind us"). One blamed the only running bounded command for the shortage. With the new text, each agent read the lines correctly and told its dispatcher that programs outside the queue hold the memory. Co-Authored-By: Claude Opus 5.5 --- skills/bounding-heavy-commands/SKILL.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/skills/bounding-heavy-commands/SKILL.md b/skills/bounding-heavy-commands/SKILL.md index 28b10b2..8b51996 100644 --- a/skills/bounding-heavy-commands/SKILL.md +++ b/skills/bounding-heavy-commands/SKILL.md @@ -48,7 +48,7 @@ python3 "${CLAUDE_PLUGIN_ROOT}/skills/bounding-heavy-commands/scripts/bounded_ru | `--label ` | A name for the row lines of a measurement, such as `unit-tests`. Use letters, digits, `.`, `_`, and `-`, and start with a letter or a digit. It changes nothing else. | | `--check` | Run no command. Print the state of the queue now, and for each `--memory`, whether a command of that budget would start now. It takes only `--memory`, which you can repeat. It holds no slot, and its exit code is 0. §The record says how to choose a row with it. | -The wrapper does four things in sequence. It waits for its turn in the queue. It waits until the memory of its budget is free. It runs the command. It prints the budget and the measured peak on the error stream. For the wait, the free memory is the memory that the machine has free now, less the part of their budgets that the running bounded commands do not use yet, less a headroom for the programs outside the queue. The headroom is a tenth of the memory of the machine, and at least 1 GiB. The budget and the headroom together never need more than the memory of the queue, so a budget of the whole queue does not wait for a headroom. While another bounded command runs, the wait has no time limit: the command does not start before its memory is free, and the wrapper prints a line at intervals while it waits. When the wrapper finds no other bounded command running, nothing that it can count will free memory, so it does not wait. It prints `bounded-run: no-memory budget=M free=M headroom=M fits=M`, releases its slots, and exits with code 75 without starting the command (§Rules for the wait). Each line that the wrapper prints starts with `bounded-run:`. A measurement also prints one row line, in a fixed form: +The wrapper does four things in sequence. It waits for its turn in the queue. It waits until the memory of its budget is free. While it waits for memory, it gives its slots back and leaves the queue, so that a smaller command whose budget fits can start. When the memory is free again, it takes its turn in the queue again. It runs the command. It prints the budget and the measured peak on the error stream. For the wait, the free memory is the memory that the machine has free now, less the part of their budgets that the running bounded commands do not use yet, less a headroom for the programs outside the queue. The headroom is a tenth of the memory of the machine, and at least 1 GiB. The budget and the headroom together never need more than the memory of the queue, so a budget of the whole queue does not wait for a headroom. While another bounded command runs, the wait has no time limit: the command does not start before its memory is free, and the wrapper prints a line at intervals while it waits. When the wrapper finds no other bounded command running, nothing that it can count will free memory, so it does not wait. It prints `bounded-run: no-memory budget=M free=M headroom=M fits=M`, releases its slots, and exits with code 75 without starting the command (§Rules for the wait). Each line that the wrapper prints starts with `bounded-run:`. A measurement also prints one row line, in a fixed form: ``` bounded-run: row label= cpus= budget=M peak=M held=M|- exact= exit= @@ -61,10 +61,11 @@ A check prints its lines in a fixed form too: ``` bounded-run: check slots= slot=M free-slots= free=M unused=M cpus= cpus-used= cpus-headroom= cpus-free= line= fits=M bounded-run: check held unit= budget=M used=M cpus= age=s dir= command= +bounded-run: check waiting unit= for=memory budget=M slots= age=s dir= command= bounded-run: check budget=M slots= headroom=M starts= ``` -The first line gives the slots of the queue and their size, the slots that are free now, the free memory, the unused budgets of the running commands, the processors of the machine, the processors that the running bounded commands can use together (`cpus-used`), the processors that the check keeps for the programs outside the queue (`cpus-headroom`, a tenth of `cpus`, rounded up), and the processors that are left for a new bounded command (`cpus-free`: `cpus` less `cpus-headroom` and `cpus-used`, and at least 0). `line=busy` means that a command waits in the line. `fits` is the largest budget whose memory, with its own headroom, is free now. It does not count the slots or the line; `starts` counts all three. Each `held` line is one bounded command that runs now: its budget, the memory it uses, the processors it can use, how long it has run, its working directory, and its command. The held lines, `cpus-used`, and `cpus-free` need the hard cap, so without it there are no held lines and `cpus-used` and `cpus-free` are `-`. Each `budget` line is one `--memory`, in the order given: the slots that it needs, the headroom that the wait keeps free beside that budget, and `starts=yes` when a command of that budget would start now. A `-` means that the check cannot read or count the value at that moment. +The first line gives the slots of the queue and their size, the slots that are free now, the free memory, the unused budgets of the running commands, the processors of the machine, the processors that the running bounded commands can use together (`cpus-used`), the processors that the check keeps for the programs outside the queue (`cpus-headroom`, a tenth of `cpus`, rounded up), and the processors that are left for a new bounded command (`cpus-free`: `cpus` less `cpus-headroom` and `cpus-used`, and at least 0). `line=busy` means that a command waits in the line. `fits` is the largest budget whose memory, with its own headroom, is free now. It does not count the slots or the line; `starts` counts all three. Each `held` line is one bounded command that runs now: its budget, the memory it uses, the processors it can use, how long it has run, its working directory, and its command. The held lines, `cpus-used`, and `cpus-free` need the hard cap, so without it there are no held lines and `cpus-used` and `cpus-free` are `-`. Each `waiting` line is one bounded command that waits for free memory: its budget, the slots that it needs, how long it has waited, its working directory, and its command. It holds no slot and has no place in the line, so the order of the `waiting` lines is not an order of the queue, and a smaller command whose budget fits starts before them. A `waiting` line means that the machine is short of memory, not of slots: the programs outside the queue hold the memory that its budget needs, together with the running commands of the `held` lines. It starts when that memory comes free, or exits 75 when no bounded command runs any more. Each `budget` line is one `--memory`, in the order given: the slots that it needs, the headroom that the wait keeps free beside that budget, and `starts=yes` when a command of that budget would start now. A `-` means that the check cannot read or count the value at that moment. The exit code of the wrapper is the command's exit code. Exit code 125, with a `bounded-run:` line that names a fault and with no line for the budget and the peak, means that the wrapper itself failed (see §When the wrapper cannot run). @@ -158,14 +159,14 @@ A command that the project adds later gets its entry when an agent first needs i ## Rules for the wait -A long wait means that the machine is full. It is never a fault to repair: the queue has no state that can go wrong between runs, and an agent that repairs it breaks it. Waiting commands take turns in a line. Only the command at the head of the line tries for slots, so the smaller commands behind it do not take the slots that it waits for. The line has no fixed order, and a head that sees no slot come free for 5 minutes lets the commands behind it try first. So a small command can wait behind a large one, also when a slot is free. Unless an instruction says otherwise (§When an instruction sends a command around the wrapper), the only correct moves are to keep the run in the queue and to tell the user that the machine is full and the command is queued. After a long wait, run the wrapper with `--check`, and in that message name each command of its `held` lines, with its directory and its age, so that the user can see what holds the machine. A subagent tells its dispatcher in its report. +A long wait means that the machine is full. It is never a fault to repair: the queue has no state that can go wrong between runs, and an agent that repairs it breaks it. Waiting commands take turns in a line. Only the command at the head of the line tries for slots, so the smaller commands behind it do not take the slots that it waits for. The line has no fixed order, and a head that sees no slot come free for 5 minutes lets the commands behind it try first. So a small command can wait behind a large one, also when a slot is free. A command that waits for free memory is not in the line and holds no slot (§The wrapper). Unless an instruction says otherwise (§When an instruction sends a command around the wrapper), the only correct moves are to keep the run in the queue and to tell the user that the machine is full and the command is queued. After a long wait, run the wrapper with `--check`, and in that message name each command of its `held` and `waiting` lines, with its directory and its age, so that the user can see what holds the machine. When the check has `waiting` lines, also say that the programs outside the queue hold the memory, not the slots. A subagent tells its dispatcher in its report. - **Run a bounded command in the background, with no short timeout around it.** It can wait in the queue for minutes before it starts. Where the harness requires a timeout, use the longest one that it permits. When that timeout ends a run, start the run again through the wrapper, and say in your report that the timeout ended the run and that the code did not fail. A timeout that you calculate from the run time of the command does not count the wait. A timeout that fires during the wait kills a command that never ran, and the result reads like a failure of the code. - **Plan the time as if the commands run one after the other.** Two bounded commands that start together are safe, because the queue puts them in sequence when the machine cannot hold both. So a parallel start does not always save time, and a promise to the user that depends on it is a guess. - **Do not lower a budget to start sooner.** A smaller budget does not make the command smaller. It moves the failure inside the command: on Linux as a kill at the hard cap, elsewhere as the swap that the queue exists to prevent. A row at a smaller `--cpus` is not a lower budget: it is a smaller command, measured, and it is the way to start sooner (§The record). - **Do not run the plain command because the queue is slow.** A slow queue means the machine is full. The plain command is how it freezes. This includes a narrower form of the command: one test file outside the queue is a command that the queue cannot count. - **Do not set an environment variable whose name starts with `BOUNDED_RUN_`.** The usage text of the wrapper lists variables that change how the queue counts memory, where it keeps its locks, and whether the hard cap applies. They are for a person who tunes the whole machine in a shell profile, and for the tests of the wrapper. A call that sets one does not share the queue with the other sessions, or it runs with no cap. A note in your project memory that recommends one is wrong: delete the note, and tell the user that you did. -- **Do not touch the lock files of the wrapper.** The wrapper keeps them in a directory of its own. Do not look for that directory. Do not delete, move, or edit a file in it, and do not look for the process that holds one of these lock files. The lock files are empty and permanent. They hold no process ID. The kernel holds each lock and releases it at the moment its holder stops, also when the holder crashes, so a stale lock cannot exist. A deleted lock file makes two commands hold one slot. The `held` lines of `--check` are the way to name what holds the queue: do not look further, and never stop a held command, because it belongs to another session or to the user. +- **Do not touch the lock files of the wrapper.** The wrapper keeps them in a directory of its own. Do not look for that directory. Do not delete, move, or edit a file in it, and do not look for the process that holds one of these lock files. The lock files hold no process ID. The kernel holds each lock and releases it at the moment its holder stops, also when the holder crashes, so a stale lock cannot exist. A deleted lock file makes two commands hold one slot. The `held` and `waiting` lines of `--check` are the way to name what holds the queue and what waits for memory: do not look further, and never stop a held or waiting command, because it belongs to another session or to the user. - **Do not sleep and retry around a collision, not even one time.** A port in use or a tool's lock means two commands wanted one resource. Add the resource to the record as an `--exclusive` name and run through the wrapper. A subagent adds the option to its own command line and names the resource in its report, so that its dispatcher corrects the record. Do not stop the process that holds the resource: it belongs to a different session. When a run with the `--exclusive` name fails on the same collision, the holder did not go through the wrapper or belongs to a different repository: tell the user, or your dispatcher, what holds the resource, and wait for the answer. ### After exit code 75 @@ -192,7 +193,7 @@ A subagent does not see your project memory. It knows the bounded commands only On a machine on which the wrapper can run, every dispatch prompt for a subagent that will run a heavy command MUST include: 1. The bounded command line for each heavy command, copied from the record, with the `--cpus` and the budget of its default row. When its entry has more than one row, also the `--cpus` and the budget of each row, and its check line with one `--memory` for each row. -2. This rule, in these words: "Run the bounded commands as written. You can make three changes: the `--cpus` and the `--memory` of a row that this prompt gives for the command, a narrower argument at the end of the line, such as one test file, and one more `--exclusive ` option after a collision on a port or a lock, which you name in your report. When this prompt gives a command more than one row, run its check line before each run. When `cpus-free` is `-`, use the largest row with `starts=yes`. Otherwise, use the largest row with `starts=yes` whose `--cpus` is not more than `cpus-free`, and when no such row fits, use the smallest row with `starts=yes`. When no row has `starts=yes`, use the default row. Do not run the plain form of a bounded command or of a part of it. For a heavy command that is not in this prompt, use the same wrapper with no `--memory` option. If the file of the wrapper does not exist, stop and report that. Set no environment variable whose name starts with `BOUNDED_RUN_`. A command can wait in the queue for minutes before it starts, so run it in the background and do not put a short timeout around it. A long wait means that the machine is full; it is not a fault to repair. Exit code 75 with a `bounded-run: no-memory` line means that the command did not start, and that the same budget gives 75 again: run the largest row of this prompt whose budget is not more than `fits`, or, when no row fits, report the line. Say in your report how long you waited. After a long wait, run the same wrapper with `--check`, and name in your report each command of its `held` lines, with its directory and its age." +2. This rule, in these words: "Run the bounded commands as written. You can make three changes: the `--cpus` and the `--memory` of a row that this prompt gives for the command, a narrower argument at the end of the line, such as one test file, and one more `--exclusive ` option after a collision on a port or a lock, which you name in your report. When this prompt gives a command more than one row, run its check line before each run. When `cpus-free` is `-`, use the largest row with `starts=yes`. Otherwise, use the largest row with `starts=yes` whose `--cpus` is not more than `cpus-free`, and when no such row fits, use the smallest row with `starts=yes`. When no row has `starts=yes`, use the default row. Do not run the plain form of a bounded command or of a part of it. For a heavy command that is not in this prompt, use the same wrapper with no `--memory` option. If the file of the wrapper does not exist, stop and report that. Set no environment variable whose name starts with `BOUNDED_RUN_`. A command can wait in the queue for minutes before it starts, so run it in the background and do not put a short timeout around it. A long wait means that the machine is full; it is not a fault to repair. Exit code 75 with a `bounded-run: no-memory` line means that the command did not start, and that the same budget gives 75 again: run the largest row of this prompt whose budget is not more than `fits`, or, when no row fits, report the line. Say in your report how long you waited. After a long wait, run the same wrapper with `--check`, and name in your report each command of its `held` and `waiting` lines, with its directory and its age. A `waiting` line means that the programs outside the queue hold the memory, not the slots: say so in your report." 3. Each command that an instruction exempts from the wrapper, in its plain form, in a list of its own, with the instruction that exempts it. The rule of item 2 does not apply to that list. An instruction of the user that covers one run goes to the one subagent that does that run, and to no other. Do not dispatch fewer subagents to protect the machine. The queue protects it. Subagents that wait in the queue cost time, not memory. On a machine on which the wrapper cannot run, the next section says how to dispatch. From 282db8de76ca7e08efd91bb55d88d8570af56e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:18:21 +0100 Subject: [PATCH 03/10] fix(skills/bounding-heavy-commands): keep a live wait file that takes a stale name A waiter puts its wait file at the same name each time it gives back its slots. A check can lock a stale file at that name, or create an empty one there in the moment after the waiter removed its old file. The waiter's rename can then put its new, held file at the name before the check removes the name, so the check deleted the live file and the waiter dropped out of `--check`. The check now removes the name only when the file at the name is still the file that it locked. Otherwise it skips the name in this pass. Run files need no such change: each unit makes its run file once and never makes it again under the same name. Co-Authored-By: Claude Opus 5.5 --- .../scripts/bounded_run.py | 11 +++++++---- .../scripts/tests/test_bounded_run.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/skills/bounding-heavy-commands/scripts/bounded_run.py b/skills/bounding-heavy-commands/scripts/bounded_run.py index cbd8023..4b40bc5 100755 --- a/skills/bounding-heavy-commands/scripts/bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/bounded_run.py @@ -570,10 +570,13 @@ def memory_waiters(directory: str, now: Callable[[], float] = time.time) -> List path = os.path.join(directory, name) descriptor = _try_lock(path) if descriptor is not None: - try: - os.unlink(path) - except OSError: - pass + # A waiter puts a new wait file at the same name each time it gives back its slots, so the name can hold a + # new, held file by now. + if _is_the_file_at(path, descriptor): + try: + os.unlink(path) + except OSError: + pass os.close(descriptor) continue try: diff --git a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py index e330192..c1559fd 100644 --- a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py @@ -765,6 +765,24 @@ def test_a_wait_file_that_no_process_holds_is_removed_and_not_named(self): self.assertEqual(self.waiters(), []) self.assertEqual(os.listdir(self.directory), []) + def test_a_new_wait_file_that_takes_the_name_after_the_lock_is_kept_and_named_on_the_next_read(self): + unit = "bounded-run-5-000005" + path = os.path.join(self.directory, bounded_run.wait_file_name(unit)) + real_try_lock = bounded_run._try_lock + + def try_lock_then_replace(name): + # The waiter writes its new wait file after the reader locked the old one. + descriptor = real_try_lock(name) + if name == path: + self.mark(unit) + return descriptor + + open(path, "w").close() + with mock.patch.object(bounded_run, "_try_lock", try_lock_then_replace): + self.assertEqual(self.waiters(), []) + self.assertTrue(os.path.exists(path)) + self.assertEqual(self.waiters(), ["unit=%s for=memory budget=6144M slots=3 age=42s dir=/work/project command=make test" % unit]) + def test_the_wait_file_holds_no_process_id(self): self.mark("bounded-run-5-000005") text = Path(self.directory, bounded_run.wait_file_name("bounded-run-5-000005")).read_text() From b2f088b9ed7f8950e3c72753f0d5919d39460b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:23:30 +0100 Subject: [PATCH 04/10] fix(skills/bounding-heavy-commands): keep the wait age of a command across its waits A waiter wrote a new start time to its wait file each time it gave back its slots. When the memory went away again after the slots came back, the `age` on its `check waiting` line started again at 0, and a command that waited for a long time looked new. The wrapper now keeps the time of its first wait for memory and writes it to each new wait file. Co-Authored-By: Claude Opus 5.5 --- .../scripts/bounded_run.py | 6 ++++- .../scripts/tests/test_bounded_run.py | 22 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/skills/bounding-heavy-commands/scripts/bounded_run.py b/skills/bounding-heavy-commands/scripts/bounded_run.py index 4b40bc5..2606e91 100755 --- a/skills/bounding-heavy-commands/scripts/bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/bounded_run.py @@ -1682,6 +1682,8 @@ def read_running() -> Optional[int]: working = "-" reservations: List[Reservation] = [] waiting: List[int] = [] + # The age on the check line counts from the first wait for memory, also after the slots came back between two waits. + waiting_since: List[float] = [] def take_slots() -> None: while waiting: @@ -1692,8 +1694,10 @@ def give_back_slots() -> None: while reservations: reservations.pop().release() log("giving back %d slot(s) while waiting for free memory" % needed) + if not waiting_since: + waiting_since.append(time.time()) if not waiting: - waiting.append(write_wait_file(directory, unit, wait_budget, needed, working, options.command)) + waiting.append(write_wait_file(directory, unit, wait_budget, needed, working, options.command, now=lambda: waiting_since[0])) reported: Optional[int] = None take_slots() diff --git a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py index c1559fd..802db8f 100644 --- a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py @@ -1,6 +1,7 @@ from __future__ import annotations import errno +import json import os import re import select @@ -2214,6 +2215,27 @@ def test_a_scope_that_never_appears_frees_the_memory_check_after_a_time_limit(se self.assertEqual(free, {"before the scope": False, "before the limit": False, "at the start": True}) self.assertLess(time.monotonic() - started, 5) + def test_the_age_of_a_waiter_counts_from_its_first_wait_for_memory(self): + starts = [] + + def wait(*arguments, **keywords): + for _ in range(2): + keywords["release"]() + path = os.path.join(self.directory, [name for name in os.listdir(self.directory) if name.startswith("wait-")][0]) + starts.append(json.loads(Path(path).read_text())["since"]) + time.sleep(0.05) + keywords["hold"]() + + options = bounded_run.Options() + options.memory = "1G" + options.command = [sys.executable, "-c", "pass"] + with mock.patch.object(bounded_run, "probe_cap", return_value=None), \ + mock.patch.object(bounded_run, "wait_for_memory", wait), \ + mock.patch.object(bounded_run, "run_command", return_value=(0, 0)): + self.assertEqual(bounded_run.bounded(options, self.environ), 0) + self.assertEqual(len(starts), 2) + self.assertEqual(starts[0], starts[1]) + def test_the_wait_counts_no_unused_budget_without_a_cap(self): self.assertIsNone(self.wait_arguments(None)["read_outstanding"]()) From 1ae738a6a50eac315102e686781f674f7f2c16f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:23:30 +0100 Subject: [PATCH 05/10] fix(skills/bounding-heavy-commands): say what a deleted wait file does The rule about the lock files said that a deleted lock file makes two commands hold one slot. That is true of a slot file only. A deleted wait file hides a command that waits for memory from `--check`. Co-Authored-By: Claude Opus 5.5 --- skills/bounding-heavy-commands/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/bounding-heavy-commands/SKILL.md b/skills/bounding-heavy-commands/SKILL.md index 8b51996..e7cb056 100644 --- a/skills/bounding-heavy-commands/SKILL.md +++ b/skills/bounding-heavy-commands/SKILL.md @@ -166,7 +166,7 @@ A long wait means that the machine is full. It is never a fault to repair: the q - **Do not lower a budget to start sooner.** A smaller budget does not make the command smaller. It moves the failure inside the command: on Linux as a kill at the hard cap, elsewhere as the swap that the queue exists to prevent. A row at a smaller `--cpus` is not a lower budget: it is a smaller command, measured, and it is the way to start sooner (§The record). - **Do not run the plain command because the queue is slow.** A slow queue means the machine is full. The plain command is how it freezes. This includes a narrower form of the command: one test file outside the queue is a command that the queue cannot count. - **Do not set an environment variable whose name starts with `BOUNDED_RUN_`.** The usage text of the wrapper lists variables that change how the queue counts memory, where it keeps its locks, and whether the hard cap applies. They are for a person who tunes the whole machine in a shell profile, and for the tests of the wrapper. A call that sets one does not share the queue with the other sessions, or it runs with no cap. A note in your project memory that recommends one is wrong: delete the note, and tell the user that you did. -- **Do not touch the lock files of the wrapper.** The wrapper keeps them in a directory of its own. Do not look for that directory. Do not delete, move, or edit a file in it, and do not look for the process that holds one of these lock files. The lock files hold no process ID. The kernel holds each lock and releases it at the moment its holder stops, also when the holder crashes, so a stale lock cannot exist. A deleted lock file makes two commands hold one slot. The `held` and `waiting` lines of `--check` are the way to name what holds the queue and what waits for memory: do not look further, and never stop a held or waiting command, because it belongs to another session or to the user. +- **Do not touch the lock files of the wrapper.** The wrapper keeps them in a directory of its own. Do not look for that directory. Do not delete, move, or edit a file in it, and do not look for the process that holds one of these lock files. The lock files hold no process ID. The kernel holds each lock and releases it at the moment its holder stops, also when the holder crashes, so a stale lock cannot exist. A deleted slot file makes two commands hold one slot, and a deleted wait file hides a command that waits for memory from `--check`. The `held` and `waiting` lines of `--check` are the way to name what holds the queue and what waits for memory: do not look further, and never stop a held or waiting command, because it belongs to another session or to the user. - **Do not sleep and retry around a collision, not even one time.** A port in use or a tool's lock means two commands wanted one resource. Add the resource to the record as an `--exclusive` name and run through the wrapper. A subagent adds the option to its own command line and names the resource in its report, so that its dispatcher corrects the record. Do not stop the process that holds the resource: it belongs to a different session. When a run with the `--exclusive` name fails on the same collision, the holder did not go through the wrapper or belongs to a different repository: tell the user, or your dispatcher, what holds the resource, and wait for the answer. ### After exit code 75 From d39bd630423abf9dac92e5ae99505f143003e728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:28:11 +0100 Subject: [PATCH 06/10] fix(skills/bounding-heavy-commands): wait for the wait file in the memory-waiter tests The test helper waited for the "; waiting" line of the large command. The wrapper prints that line before it gives back its slots and writes its wait file, so a check that ran at once could miss the `waiting` line. The helper now also waits until the wait file exists. Co-Authored-By: Claude Opus 5.5 --- .../scripts/tests/test_bounded_run.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py index 802db8f..07bbf93 100644 --- a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py @@ -3179,6 +3179,11 @@ def memory_waiter(self): self.wait_for_event("start", "holder") large = self.worker("large", "6G", hold="0") # does not start: its budget never fits in the free memory self.wait_for_stderr(large, "; waiting") + # The wrapper logs the wait before it gives back its slots and writes its wait file. + deadline = time.time() + 60 + while not any(name.startswith("wait-") and name.endswith(".lock") for name in os.listdir(self.environ["BOUNDED_RUN_LOCK_DIR"])): + self.assertLess(time.time(), deadline, "the large command wrote no wait file") + time.sleep(0.02) return release, holder, large def test_a_command_that_waits_for_memory_does_not_stop_a_smaller_command_that_fits(self): From e20798e21752b91ef807fe1a9bdc4e475066a73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:35:40 +0100 Subject: [PATCH 07/10] fix(skills/bounding-heavy-commands): correct two promises of the memory wait's docstring The docstring of `wait_for_memory` said that the call returns at once when the free memory cannot be read. Since the wait gives back its slots, the call first takes them back with `hold`, which can wait in the line. It also said that NoMemory leaves the slots as they are, which a caller can read as "still held". NoMemory can come after the call gave the slots back. The docstring now says that the call returns without a wait for memory when the free memory cannot be read, and that NoMemory can come after the call gave the slots back. Co-Authored-By: Claude Opus 5.5 --- skills/bounding-heavy-commands/scripts/bounded_run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skills/bounding-heavy-commands/scripts/bounded_run.py b/skills/bounding-heavy-commands/scripts/bounded_run.py index 2606e91..022b3a0 100755 --- a/skills/bounding-heavy-commands/scripts/bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/bounded_run.py @@ -793,9 +793,10 @@ def wait_for_memory( There is no time limit while another bounded command runs, so the call can block for as long as the machine is full. Raises NoMemory, with lock released, when none runs, because then nothing in the queue frees memory. read_running gives the number of bounded commands that run, or None where it is not known: the call waits. - Returns at once when the free memory cannot be read. read_outstanding gives None where the unused budgets cannot be counted. + When the free memory cannot be read, the call returns without a wait for memory. + read_outstanding gives None where the unused budgets cannot be counted. The call returns with lock held and the slots held: the caller calls unlock once the new command counts in read_outstanding. - NoMemory leaves the slots as they are. + NoMemory can come after the call gave the slots back. """ def read() -> Tuple[Optional[int], Optional[int]]: # A running command that grows or shrinks during the reads moves both values. The larger of the two unused From ce84ad92866549857ad62b9834458d1e1dda3913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:39:59 +0100 Subject: [PATCH 08/10] fix(skills/bounding-heavy-commands): keep the slots of a command that waits for memory Giving back the slots while a command waits for memory breaks the fairness of the line. The large waiter loses its turn, and smaller commands that fit can take the memory each time it comes free, so the large one can wait without end. The case that blocked the queue with no bounded command running already ends with exit 75. The wait that is left lasts until a running bounded command frees its memory, and holding the slots there keeps the line fair. A command that waits for memory now keeps its slots and its turn, as before this branch. `wait_for_memory` has the contract and tests that it had on main, plus one `on_wait` callback that runs before each sleep. The wrapper uses it to write its wait file once, and removes the file when the wait ends. So `--check` still prints a `waiting` line, and its `slots` are the slots that the waiter holds. Because the wait file is now written once for each wait, the start time needs no copy and the reader needs no check that the name still holds the file that it locked. The process test now checks that `free-slots` counts the slots of the waiter beside a holder, and the test that a smaller command starts before the waiter is gone. Co-Authored-By: Claude Opus 5.5 --- .../scripts/bounded_run.py | 89 +++++------- .../scripts/tests/test_bounded_run.py | 135 +++++------------- 2 files changed, 68 insertions(+), 156 deletions(-) diff --git a/skills/bounding-heavy-commands/scripts/bounded_run.py b/skills/bounding-heavy-commands/scripts/bounded_run.py index 022b3a0..83d53d4 100755 --- a/skills/bounding-heavy-commands/scripts/bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/bounded_run.py @@ -64,9 +64,8 @@ cpus-headroom and cpus-used, and at least 0. starts=yes means that a run of that budget would start now. It is not a reservation. fits is the largest budget whose memory, with its own headroom, is free now, in steps of 256M, -or 0M when none is; it does not count slots or the line. A command that -waits for memory holds none of the slots of its waiting line, so a smaller -command whose budget fits can start before it. +or 0M when none is; it does not count slots or the line. A waiting line is +a command that holds its slots and waits for memory. A normal call sets none of the environment variables below. A person can set the tuning values in the profile of the shell, so that every session @@ -528,7 +527,8 @@ def write_wait_file( ) -> int: """Writes the wait file of the unit and holds it locked. Returns its descriptor for remove_wait_file. - The file names the budget, the slots, the directory, and the command, for memory_waiters to read. It holds no process ID. + The file names the budget, the slots that the command holds, the directory, and the command, for memory_waiters to read. + It holds no process ID. """ path = os.path.join(directory, wait_file_name(unit)) partial = path + ".new" @@ -570,13 +570,10 @@ def memory_waiters(directory: str, now: Callable[[], float] = time.time) -> List path = os.path.join(directory, name) descriptor = _try_lock(path) if descriptor is not None: - # A waiter puts a new wait file at the same name each time it gives back its slots, so the name can hold a - # new, held file by now. - if _is_the_file_at(path, descriptor): - try: - os.unlink(path) - except OSError: - pass + try: + os.unlink(path) + except OSError: + pass os.close(descriptor) continue try: @@ -781,22 +778,16 @@ def wait_for_memory( lock: Callable[[], None] = lambda: None, unlock: Callable[[], None] = lambda: None, clock: Callable[[], float] = time.monotonic, - hold: Callable[[], None] = lambda: None, - release: Callable[[], None] = lambda: None, + on_wait: Callable[[], None] = lambda: None, ) -> None: """Returns when the free memory, less the unused budgets of running commands and the headroom, holds the budget. - The caller holds the slots of the command at the call. While the memory is not free, the call gives them back - with release, so that a smaller command can start with them. When the memory is free again, it takes them with - hold, with lock released, and reads the memory again before it returns. hold can wait in the line for minutes. - There is no time limit while another bounded command runs, so the call can block for as long as the machine is full. Raises NoMemory, with lock released, when none runs, because then nothing in the queue frees memory. read_running gives the number of bounded commands that run, or None where it is not known: the call waits. - When the free memory cannot be read, the call returns without a wait for memory. - read_outstanding gives None where the unused budgets cannot be counted. - The call returns with lock held and the slots held: the caller calls unlock once the new command counts in read_outstanding. - NoMemory can come after the call gave the slots back. + Returns at once when the free memory cannot be read. read_outstanding gives None where the unused budgets cannot be counted. + The call returns with lock held: the caller calls unlock once the new command counts in read_outstanding. + on_wait is called, with lock released, before each sleep of the wait. """ def read() -> Tuple[Optional[int], Optional[int]]: # A running command that grows or shrinks during the reads moves both values. The larger of the two unused @@ -814,34 +805,32 @@ def state(available: int, outstanding: Optional[int]) -> str: started = clock() next_line: Optional[float] = None - held = True while True: lock() available, outstanding = read() - fits = available is None or memory_fits(budget_mib, available, outstanding, headroom_mib) - if not fits and read_running() == 0: + if available is None: + log("cannot read the free memory; starting") + return + if memory_fits(budget_mib, available, outstanding, headroom_mib): + log(state(available, outstanding) + "; starting") + return + if read_running() == 0: # A command ends without the memory lock, so the last one can free its memory after the reads above. available, outstanding = read() - fits = available is None or memory_fits(budget_mib, available, outstanding, headroom_mib) - if not fits: - unlock() - raise NoMemory(available, outstanding) - if fits: - if held: - log("cannot read the free memory; starting" if available is None else state(available, outstanding) + "; starting") + if available is None: + log("cannot read the free memory; starting") + return + if memory_fits(budget_mib, available, outstanding, headroom_mib): + log(state(available, outstanding) + "; starting") return unlock() - hold() - held = True - continue + raise NoMemory(available, outstanding) waited = clock() - started if next_line is None or waited >= next_line: log(state(available, outstanding) + ("; waiting" if next_line is None else "; still waiting after %ds" % waited)) next_line = waited + interval_seconds unlock() - if held: - release() - held = False + on_wait() sleep(poll_seconds) @@ -1681,27 +1670,14 @@ def read_running() -> Optional[int]: working = os.getcwd() except OSError: working = "-" - reservations: List[Reservation] = [] waiting: List[int] = [] - # The age on the check line counts from the first wait for memory, also after the slots came back between two waits. - waiting_since: List[float] = [] - def take_slots() -> None: - while waiting: - remove_wait_file(directory, unit, waiting.pop()) - reservations.append(reserve(directory, count, needed, exclusive_files, settings.poll_seconds)) - - def give_back_slots() -> None: - while reservations: - reservations.pop().release() - log("giving back %d slot(s) while waiting for free memory" % needed) - if not waiting_since: - waiting_since.append(time.time()) + def mark_waiting() -> None: if not waiting: - waiting.append(write_wait_file(directory, unit, wait_budget, needed, working, options.command, now=lambda: waiting_since[0])) + waiting.append(write_wait_file(directory, unit, wait_budget, needed, working, options.command)) reported: Optional[int] = None - take_slots() + reservation = reserve(directory, count, needed, exclusive_files, settings.poll_seconds) try: try: wait_for_memory( @@ -1709,13 +1685,15 @@ def give_back_slots() -> None: headroom_mib=headroom_mib(settings.total_mib, wait_budget, most), read_available=read_available, read_outstanding=read_outstanding, read_running=read_running, lock=lock_memory, unlock=unlock_memory, - hold=take_slots, release=give_back_slots, + on_wait=mark_waiting, ) except NoMemory as stop: log("no-memory budget=%dM free=%dM headroom=%dM fits=%dM" % ( wait_budget, stop.available - (stop.outstanding or 0), headroom_mib(settings.total_mib, wait_budget, most), largest_fit_mib(stop.available, stop.outstanding, settings.total_mib, most))) raise + while waiting: + remove_wait_file(directory, unit, waiting.pop()) code, maxrss = run_command(command, child_environ, on_start) tracked = trackers[0].finish() if trackers else None held = trackers[0].held if trackers else None @@ -1732,8 +1710,7 @@ def give_back_slots() -> None: reported = read_peak_file(peak_file) while waiting: remove_wait_file(directory, unit, waiting.pop()) - while reservations: - reservations.pop().release() + reservation.release() exact = reported is not None peak = max(reported or tracked or 0, rusage_peak_mib(maxrss)) diff --git a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py index 07bbf93..d30157e 100644 --- a/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py +++ b/skills/bounding-heavy-commands/scripts/tests/test_bounded_run.py @@ -1,7 +1,6 @@ from __future__ import annotations import errno -import json import os import re import select @@ -766,24 +765,6 @@ def test_a_wait_file_that_no_process_holds_is_removed_and_not_named(self): self.assertEqual(self.waiters(), []) self.assertEqual(os.listdir(self.directory), []) - def test_a_new_wait_file_that_takes_the_name_after_the_lock_is_kept_and_named_on_the_next_read(self): - unit = "bounded-run-5-000005" - path = os.path.join(self.directory, bounded_run.wait_file_name(unit)) - real_try_lock = bounded_run._try_lock - - def try_lock_then_replace(name): - # The waiter writes its new wait file after the reader locked the old one. - descriptor = real_try_lock(name) - if name == path: - self.mark(unit) - return descriptor - - open(path, "w").close() - with mock.patch.object(bounded_run, "_try_lock", try_lock_then_replace): - self.assertEqual(self.waiters(), []) - self.assertTrue(os.path.exists(path)) - self.assertEqual(self.waiters(), ["unit=%s for=memory budget=6144M slots=3 age=42s dir=/work/project command=make test" % unit]) - def test_the_wait_file_holds_no_process_id(self): self.mark("bounded-run-5-000005") text = Path(self.directory, bounded_run.wait_file_name("bounded-run-5-000005")).read_text() @@ -956,9 +937,8 @@ def sleep(seconds): sleeps.append(seconds) now[0] += seconds * oversleep - # The last value stays, for the reading after the slots are taken again. bounded_run.wait_for_memory( - 4096, limit, 5.0, headroom_mib=headroom, read_available=lambda: readings.pop(0) if len(readings) > 1 else readings[0], + 4096, limit, 5.0, headroom_mib=headroom, read_available=lambda: readings.pop(0), read_outstanding=lambda: outstanding, sleep=sleep, clock=lambda: now[0], ) return sleeps @@ -1008,59 +988,20 @@ def read(): bounded_run.wait_for_memory( 4096, 300, 5.0, read_available=read, sleep=lambda seconds: events.append("sleep"), lock=lambda: events.append("lock"), unlock=lambda: events.append("unlock"), - hold=lambda: readings.append(5000), ) - self.assertEqual(events, ["lock", "read", "unlock", "sleep", "lock", "read", "unlock", "lock", "read"]) + self.assertEqual(events, ["lock", "read", "unlock", "sleep", "lock", "read"]) - def slot_events(self, values, running=1): + def test_calls_on_wait_before_each_sleep_with_the_lock_released(self): events = [] - readings = list(values) - - def read(): - events.append("read") - return readings.pop(0) - + readings = [1000, 1000, 5000] bounded_run.wait_for_memory( - 4096, 300, 5.0, read_available=read, read_running=lambda: running, sleep=lambda seconds: events.append("sleep"), - lock=lambda: events.append("lock"), unlock=lambda: events.append("unlock"), - hold=lambda: events.append("hold"), release=lambda: events.append("release"), + 4096, 300, 5.0, read_available=lambda: readings.pop(0), sleep=lambda seconds: events.append("sleep"), + lock=lambda: events.append("lock"), unlock=lambda: events.append("unlock"), on_wait=lambda: events.append("wait"), ) - return events - - def test_keeps_the_slots_when_the_memory_is_free_at_once(self): - self.assertEqual(self.slot_events([8000]), ["lock", "read"]) - - def test_gives_back_the_slots_while_it_waits_and_takes_them_again_before_it_reads_the_memory_that_starts_it(self): - self.assertEqual(self.slot_events([1000, 1000, 8000, 8000]), [ - "lock", "read", "unlock", "release", "sleep", - "lock", "read", "unlock", "sleep", - "lock", "read", "unlock", "hold", - "lock", "read", - ]) - - def test_gives_back_the_slots_again_when_the_memory_is_taken_while_it_waits_for_them(self): - self.assertEqual(self.slot_events([1000, 8000, 1000, 8000, 8000]), [ - "lock", "read", "unlock", "release", "sleep", - "lock", "read", "unlock", "hold", - "lock", "read", "unlock", "release", "sleep", - "lock", "read", "unlock", "hold", - "lock", "read", - ]) + self.assertEqual(events, ["lock", "unlock", "wait", "sleep", "lock", "unlock", "wait", "sleep", "lock"]) - def test_says_starting_only_when_it_holds_the_slots(self): - self.slot_events([1000, 8000, 8000]) - starting = [line for line in self.lines() if line.endswith("; starting")] - self.assertEqual(len(starting), 1, self.lines()) - - def test_stops_without_the_slots_when_the_memory_does_not_come_free_and_no_bounded_command_runs_any_more(self): - running = [1, 0] - events = [] - with self.assertRaises(bounded_run.NoMemory): - bounded_run.wait_for_memory( - 4096, 300, 5.0, read_available=lambda: 1000, read_running=lambda: running.pop(0), - sleep=lambda seconds: None, hold=lambda: events.append("hold"), release=lambda: events.append("release"), - ) - self.assertEqual(events, ["release"]) + def test_does_not_call_on_wait_when_the_memory_is_free_at_once(self): + bounded_run.wait_for_memory(4096, 300, 5.0, read_available=lambda: 8000, on_wait=lambda: self.fail("waits")) def test_reads_the_unused_budgets_before_and_after_the_free_memory(self): events = [] @@ -1081,8 +1022,8 @@ def read_outstanding(): def test_counts_the_larger_of_the_unused_budgets_read_before_and_after_the_free_memory(self): for first, second in ((1000, 3000), (3000, 1000)): with self.subTest(first=first, second=second): - unused = [first, second, 0, 0, 0, 0] - readings = [4000, 8000, 8000] + unused = [first, second, 0, 0] + readings = [4000, 8000] sleeps = [] bounded_run.wait_for_memory( 2048, 300, 5.0, read_available=lambda: readings.pop(0), read_outstanding=lambda: unused.pop(0), @@ -1115,7 +1056,7 @@ def test_stops_when_the_memory_is_not_free_and_no_bounded_command_runs(self): self.assertEqual(raised.exception.available, 1000) def test_waits_while_another_bounded_command_runs(self): - readings = [1000, 5000, 5000] + readings = [1000, 5000] sleeps = [] bounded_run.wait_for_memory(4096, 300, 5.0, read_available=lambda: readings.pop(0), read_running=lambda: 1, sleep=sleeps.append) @@ -2215,26 +2156,29 @@ def test_a_scope_that_never_appears_frees_the_memory_check_after_a_time_limit(se self.assertEqual(free, {"before the scope": False, "before the limit": False, "at the start": True}) self.assertLess(time.monotonic() - started, 5) - def test_the_age_of_a_waiter_counts_from_its_first_wait_for_memory(self): - starts = [] + def test_the_wait_file_names_the_held_slots_while_the_command_waits_and_is_gone_when_it_starts(self): + seen = [] def wait(*arguments, **keywords): - for _ in range(2): - keywords["release"]() - path = os.path.join(self.directory, [name for name in os.listdir(self.directory) if name.startswith("wait-")][0]) - starts.append(json.loads(Path(path).read_text())["since"]) - time.sleep(0.05) - keywords["hold"]() + keywords["on_wait"]() + keywords["on_wait"]() + seen.append(bounded_run.memory_waiters(self.directory)) + + def fake_run_command(command, environ, on_start=None): + seen.append(bounded_run.memory_waiters(self.directory)) + return 0, 0 options = bounded_run.Options() options.memory = "1G" options.command = [sys.executable, "-c", "pass"] with mock.patch.object(bounded_run, "probe_cap", return_value=None), \ mock.patch.object(bounded_run, "wait_for_memory", wait), \ - mock.patch.object(bounded_run, "run_command", return_value=(0, 0)): + mock.patch.object(bounded_run, "run_command", fake_run_command): self.assertEqual(bounded_run.bounded(options, self.environ), 0) - self.assertEqual(len(starts), 2) - self.assertEqual(starts[0], starts[1]) + self.assertEqual(len(seen[0]), 1) + self.assertIn(" for=memory budget=1024M slots=1 ", seen[0][0]) + self.assertEqual(seen[1], []) + self.assertEqual([name for name in os.listdir(self.directory) if name.startswith("wait-")], []) def test_the_wait_counts_no_unused_budget_without_a_cap(self): self.assertIsNone(self.wait_arguments(None)["read_outstanding"]()) @@ -3169,9 +3113,9 @@ def test_a_waiting_command_holds_no_slot_and_is_not_overtaken(self): def memory_waiter(self): """Starts a holder, then a large command whose budget does not fit in the free memory, with five slots of 2 GiB. - Each wrapper reads 6000 MiB of free memory and keeps a headroom of 1228 MiB. The large command needs 3 slots, - and the 4 GiB command needs 2 of the 4 slots that the holder leaves. Returns the release file, the holder, and the - large command, once the large command waits for memory. + Each wrapper reads 6000 MiB of free memory and keeps a headroom of 1228 MiB, so the large command needs 3 slots and + waits for memory while the holder runs. Returns the release file, the holder, and the large command, once the large + command waits for memory. """ self.environ.update({"BOUNDED_RUN_TOTAL_MIB": "12288", "BOUNDED_RUN_FREE_MIB": "6000"}) release = os.path.join(self.base.name, "release") @@ -3179,33 +3123,24 @@ def memory_waiter(self): self.wait_for_event("start", "holder") large = self.worker("large", "6G", hold="0") # does not start: its budget never fits in the free memory self.wait_for_stderr(large, "; waiting") - # The wrapper logs the wait before it gives back its slots and writes its wait file. + # The wrapper logs the wait before it writes its wait file. deadline = time.time() + 60 while not any(name.startswith("wait-") and name.endswith(".lock") for name in os.listdir(self.environ["BOUNDED_RUN_LOCK_DIR"])): self.assertLess(time.time(), deadline, "the large command wrote no wait file") time.sleep(0.02) return release, holder, large - def test_a_command_that_waits_for_memory_does_not_stop_a_smaller_command_that_fits(self): - release, holder, large = self.memory_waiter() - small = self.worker("small", "4G", hold="0") # ends at once; the test waits for it to end - self.assertEqual(small.wait(timeout=60), 0) - self.assertIsNone(holder.poll()) - Path(release).write_text("") - self.assertEqual(holder.wait(timeout=60), 0) - self.assertEqual(large.wait(timeout=60), bounded_run.EXIT_NO_MEMORY) - self.assertNotIn(["start", "large"], [line.split()[:2] for line in self.read_events()]) - self.assertEqual([name for name in os.listdir(self.environ["BOUNDED_RUN_LOCK_DIR"]) if name.startswith("wait-")], []) - def test_a_check_names_the_command_that_waits_for_memory(self): release, holder, large = self.memory_waiter() done = subprocess.run([sys.executable, WRAPPER, "--check"], env=self.environ, capture_output=True, text=True, timeout=60) self.assertRegex(done.stderr, r"\nbounded-run: check waiting unit=bounded-run-%d-[0-9a-f]{6} for=memory budget=6144M slots=3 age=\d+s dir=%s command=%s -c import os, sys, time def event" % ( large.pid, re.escape(os.getcwd()), re.escape(sys.executable))) - self.assertIn(" free-slots=4 ", done.stderr) + # The holder holds 1 slot and the waiter 3 of the 5. + self.assertIn(" free-slots=1 ", done.stderr) Path(release).write_text("") - holder.wait(timeout=60) - large.wait(timeout=60) + self.assertEqual(holder.wait(timeout=60), 0) + self.assertEqual(large.wait(timeout=60), bounded_run.EXIT_NO_MEMORY) + self.assertEqual([name for name in os.listdir(self.environ["BOUNDED_RUN_LOCK_DIR"]) if name.startswith("wait-")], []) def test_a_hard_kill_of_the_wrapper_frees_the_slots_at_once(self): holder = self.worker("holder", "4G", hold="300") # runs until the kill below ends it From fdbf4b960b1b8a48252378a621935ededaa59cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:40:16 +0100 Subject: [PATCH 09/10] docs(skills/bounding-heavy-commands): a waiting command keeps its slots The skill said that a command that waits for memory gives its slots back, leaves the line, and lets smaller commands start before it. The wrapper no longer does that. The skill now says that a `waiting` line is a command that had its turn and keeps its slots while it waits, so it does not lose its turn to a command behind it. It waits for the memory that a running bounded command of a `held` line frees when it ends, because the programs outside the queue hold most of the memory. It starts when that memory is free, or exits 75 when no bounded command runs any more. The report after a long wait and the rule for subagents say that the machine is short of memory, not of slots. With this text, isolated agents that read a check with one `held` line and one `waiting` line each said that the waiting command holds its slots and keeps its turn, and that it starts when the held command ends. Co-Authored-By: Claude Opus 5.5 --- skills/bounding-heavy-commands/SKILL.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/skills/bounding-heavy-commands/SKILL.md b/skills/bounding-heavy-commands/SKILL.md index e7cb056..1d18fdf 100644 --- a/skills/bounding-heavy-commands/SKILL.md +++ b/skills/bounding-heavy-commands/SKILL.md @@ -48,7 +48,7 @@ python3 "${CLAUDE_PLUGIN_ROOT}/skills/bounding-heavy-commands/scripts/bounded_ru | `--label ` | A name for the row lines of a measurement, such as `unit-tests`. Use letters, digits, `.`, `_`, and `-`, and start with a letter or a digit. It changes nothing else. | | `--check` | Run no command. Print the state of the queue now, and for each `--memory`, whether a command of that budget would start now. It takes only `--memory`, which you can repeat. It holds no slot, and its exit code is 0. §The record says how to choose a row with it. | -The wrapper does four things in sequence. It waits for its turn in the queue. It waits until the memory of its budget is free. While it waits for memory, it gives its slots back and leaves the queue, so that a smaller command whose budget fits can start. When the memory is free again, it takes its turn in the queue again. It runs the command. It prints the budget and the measured peak on the error stream. For the wait, the free memory is the memory that the machine has free now, less the part of their budgets that the running bounded commands do not use yet, less a headroom for the programs outside the queue. The headroom is a tenth of the memory of the machine, and at least 1 GiB. The budget and the headroom together never need more than the memory of the queue, so a budget of the whole queue does not wait for a headroom. While another bounded command runs, the wait has no time limit: the command does not start before its memory is free, and the wrapper prints a line at intervals while it waits. When the wrapper finds no other bounded command running, nothing that it can count will free memory, so it does not wait. It prints `bounded-run: no-memory budget=M free=M headroom=M fits=M`, releases its slots, and exits with code 75 without starting the command (§Rules for the wait). Each line that the wrapper prints starts with `bounded-run:`. A measurement also prints one row line, in a fixed form: +The wrapper does four things in sequence. It waits for its turn in the queue. It waits until the memory of its budget is free. It runs the command. It prints the budget and the measured peak on the error stream. For the wait, the free memory is the memory that the machine has free now, less the part of their budgets that the running bounded commands do not use yet, less a headroom for the programs outside the queue. The headroom is a tenth of the memory of the machine, and at least 1 GiB. The budget and the headroom together never need more than the memory of the queue, so a budget of the whole queue does not wait for a headroom. While another bounded command runs, the wait has no time limit: the command does not start before its memory is free, and the wrapper prints a line at intervals while it waits. When the wrapper finds no other bounded command running, nothing that it can count will free memory, so it does not wait. It prints `bounded-run: no-memory budget=M free=M headroom=M fits=M`, releases its slots, and exits with code 75 without starting the command (§Rules for the wait). Each line that the wrapper prints starts with `bounded-run:`. A measurement also prints one row line, in a fixed form: ``` bounded-run: row label= cpus= budget=M peak=M held=M|- exact= exit= @@ -65,7 +65,7 @@ bounded-run: check waiting unit= for=memory budget=M slots= age=s bounded-run: check budget=M slots= headroom=M starts= ``` -The first line gives the slots of the queue and their size, the slots that are free now, the free memory, the unused budgets of the running commands, the processors of the machine, the processors that the running bounded commands can use together (`cpus-used`), the processors that the check keeps for the programs outside the queue (`cpus-headroom`, a tenth of `cpus`, rounded up), and the processors that are left for a new bounded command (`cpus-free`: `cpus` less `cpus-headroom` and `cpus-used`, and at least 0). `line=busy` means that a command waits in the line. `fits` is the largest budget whose memory, with its own headroom, is free now. It does not count the slots or the line; `starts` counts all three. Each `held` line is one bounded command that runs now: its budget, the memory it uses, the processors it can use, how long it has run, its working directory, and its command. The held lines, `cpus-used`, and `cpus-free` need the hard cap, so without it there are no held lines and `cpus-used` and `cpus-free` are `-`. Each `waiting` line is one bounded command that waits for free memory: its budget, the slots that it needs, how long it has waited, its working directory, and its command. It holds no slot and has no place in the line, so the order of the `waiting` lines is not an order of the queue, and a smaller command whose budget fits starts before them. A `waiting` line means that the machine is short of memory, not of slots: the programs outside the queue hold the memory that its budget needs, together with the running commands of the `held` lines. It starts when that memory comes free, or exits 75 when no bounded command runs any more. Each `budget` line is one `--memory`, in the order given: the slots that it needs, the headroom that the wait keeps free beside that budget, and `starts=yes` when a command of that budget would start now. A `-` means that the check cannot read or count the value at that moment. +The first line gives the slots of the queue and their size, the slots that are free now, the free memory, the unused budgets of the running commands, the processors of the machine, the processors that the running bounded commands can use together (`cpus-used`), the processors that the check keeps for the programs outside the queue (`cpus-headroom`, a tenth of `cpus`, rounded up), and the processors that are left for a new bounded command (`cpus-free`: `cpus` less `cpus-headroom` and `cpus-used`, and at least 0). `line=busy` means that a command waits in the line. `fits` is the largest budget whose memory, with its own headroom, is free now. It does not count the slots or the line; `starts` counts all three. Each `held` line is one bounded command that runs now: its budget, the memory it uses, the processors it can use, how long it has run, its working directory, and its command. The held lines, `cpus-used`, and `cpus-free` need the hard cap, so without it there are no held lines and `cpus-used` and `cpus-free` are `-`. Each `waiting` line is one bounded command that had its turn in the queue and now waits for free memory: its budget, the slots that it holds, how long it has waited, its working directory, and its command. It keeps its slots while it waits, so it does not lose its turn to a command behind it in the line. A `waiting` line means that the machine is short of memory, not of slots: the programs outside the queue hold most of the memory, and the command waits for the memory that a running bounded command of a `held` line frees when it ends. It starts when that memory is free, or exits 75 when no bounded command runs any more. Each `budget` line is one `--memory`, in the order given: the slots that it needs, the headroom that the wait keeps free beside that budget, and `starts=yes` when a command of that budget would start now. A `-` means that the check cannot read or count the value at that moment. The exit code of the wrapper is the command's exit code. Exit code 125, with a `bounded-run:` line that names a fault and with no line for the budget and the peak, means that the wrapper itself failed (see §When the wrapper cannot run). @@ -159,7 +159,7 @@ A command that the project adds later gets its entry when an agent first needs i ## Rules for the wait -A long wait means that the machine is full. It is never a fault to repair: the queue has no state that can go wrong between runs, and an agent that repairs it breaks it. Waiting commands take turns in a line. Only the command at the head of the line tries for slots, so the smaller commands behind it do not take the slots that it waits for. The line has no fixed order, and a head that sees no slot come free for 5 minutes lets the commands behind it try first. So a small command can wait behind a large one, also when a slot is free. A command that waits for free memory is not in the line and holds no slot (§The wrapper). Unless an instruction says otherwise (§When an instruction sends a command around the wrapper), the only correct moves are to keep the run in the queue and to tell the user that the machine is full and the command is queued. After a long wait, run the wrapper with `--check`, and in that message name each command of its `held` and `waiting` lines, with its directory and its age, so that the user can see what holds the machine. When the check has `waiting` lines, also say that the programs outside the queue hold the memory, not the slots. A subagent tells its dispatcher in its report. +A long wait means that the machine is full. It is never a fault to repair: the queue has no state that can go wrong between runs, and an agent that repairs it breaks it. Waiting commands take turns in a line. Only the command at the head of the line tries for slots, so the smaller commands behind it do not take the slots that it waits for. The line has no fixed order, and a head that sees no slot come free for 5 minutes lets the commands behind it try first. So a small command can wait behind a large one, also when a slot is free. Unless an instruction says otherwise (§When an instruction sends a command around the wrapper), the only correct moves are to keep the run in the queue and to tell the user that the machine is full and the command is queued. After a long wait, run the wrapper with `--check`, and in that message name each command of its `held` and `waiting` lines, with its directory and its age, so that the user can see what holds the machine. When the check has `waiting` lines, also say that the machine is short of memory, not of slots: the programs outside the queue hold most of the memory, and the waiting commands start when the `held` commands end. A subagent tells its dispatcher in its report. - **Run a bounded command in the background, with no short timeout around it.** It can wait in the queue for minutes before it starts. Where the harness requires a timeout, use the longest one that it permits. When that timeout ends a run, start the run again through the wrapper, and say in your report that the timeout ended the run and that the code did not fail. A timeout that you calculate from the run time of the command does not count the wait. A timeout that fires during the wait kills a command that never ran, and the result reads like a failure of the code. - **Plan the time as if the commands run one after the other.** Two bounded commands that start together are safe, because the queue puts them in sequence when the machine cannot hold both. So a parallel start does not always save time, and a promise to the user that depends on it is a guess. @@ -193,7 +193,7 @@ A subagent does not see your project memory. It knows the bounded commands only On a machine on which the wrapper can run, every dispatch prompt for a subagent that will run a heavy command MUST include: 1. The bounded command line for each heavy command, copied from the record, with the `--cpus` and the budget of its default row. When its entry has more than one row, also the `--cpus` and the budget of each row, and its check line with one `--memory` for each row. -2. This rule, in these words: "Run the bounded commands as written. You can make three changes: the `--cpus` and the `--memory` of a row that this prompt gives for the command, a narrower argument at the end of the line, such as one test file, and one more `--exclusive ` option after a collision on a port or a lock, which you name in your report. When this prompt gives a command more than one row, run its check line before each run. When `cpus-free` is `-`, use the largest row with `starts=yes`. Otherwise, use the largest row with `starts=yes` whose `--cpus` is not more than `cpus-free`, and when no such row fits, use the smallest row with `starts=yes`. When no row has `starts=yes`, use the default row. Do not run the plain form of a bounded command or of a part of it. For a heavy command that is not in this prompt, use the same wrapper with no `--memory` option. If the file of the wrapper does not exist, stop and report that. Set no environment variable whose name starts with `BOUNDED_RUN_`. A command can wait in the queue for minutes before it starts, so run it in the background and do not put a short timeout around it. A long wait means that the machine is full; it is not a fault to repair. Exit code 75 with a `bounded-run: no-memory` line means that the command did not start, and that the same budget gives 75 again: run the largest row of this prompt whose budget is not more than `fits`, or, when no row fits, report the line. Say in your report how long you waited. After a long wait, run the same wrapper with `--check`, and name in your report each command of its `held` and `waiting` lines, with its directory and its age. A `waiting` line means that the programs outside the queue hold the memory, not the slots: say so in your report." +2. This rule, in these words: "Run the bounded commands as written. You can make three changes: the `--cpus` and the `--memory` of a row that this prompt gives for the command, a narrower argument at the end of the line, such as one test file, and one more `--exclusive ` option after a collision on a port or a lock, which you name in your report. When this prompt gives a command more than one row, run its check line before each run. When `cpus-free` is `-`, use the largest row with `starts=yes`. Otherwise, use the largest row with `starts=yes` whose `--cpus` is not more than `cpus-free`, and when no such row fits, use the smallest row with `starts=yes`. When no row has `starts=yes`, use the default row. Do not run the plain form of a bounded command or of a part of it. For a heavy command that is not in this prompt, use the same wrapper with no `--memory` option. If the file of the wrapper does not exist, stop and report that. Set no environment variable whose name starts with `BOUNDED_RUN_`. A command can wait in the queue for minutes before it starts, so run it in the background and do not put a short timeout around it. A long wait means that the machine is full; it is not a fault to repair. Exit code 75 with a `bounded-run: no-memory` line means that the command did not start, and that the same budget gives 75 again: run the largest row of this prompt whose budget is not more than `fits`, or, when no row fits, report the line. Say in your report how long you waited. After a long wait, run the same wrapper with `--check`, and name in your report each command of its `held` and `waiting` lines, with its directory and its age. A `waiting` line is a command that holds its slots and waits for memory, because the programs outside the queue hold most of it: say in your report that the machine is short of memory, not of slots." 3. Each command that an instruction exempts from the wrapper, in its plain form, in a list of its own, with the instruction that exempts it. The rule of item 2 does not apply to that list. An instruction of the user that covers one run goes to the one subagent that does that run, and to no other. Do not dispatch fewer subagents to protect the machine. The queue protects it. Subagents that wait in the queue cost time, not memory. On a machine on which the wrapper cannot run, the next section says how to dispatch. From 1fedfd5f9083fb4b193763be78b45b80f30de30d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:44:50 +0100 Subject: [PATCH 10/10] fix(skills/bounding-heavy-commands): a waiter waits for a running command, which has a held line only under the cap The skill said that a waiting command waits for the memory of a running command of a `held` line, and that the waiting commands start when the `held` commands end. Without the hard cap there are no `held` lines, but `--check` still prints the `waiting` lines. The skill now says that the waiter waits for a running bounded command, and that this command has a `held` line where the hard cap is available. Co-Authored-By: Claude Opus 5.5 --- skills/bounding-heavy-commands/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skills/bounding-heavy-commands/SKILL.md b/skills/bounding-heavy-commands/SKILL.md index 1d18fdf..5dbd942 100644 --- a/skills/bounding-heavy-commands/SKILL.md +++ b/skills/bounding-heavy-commands/SKILL.md @@ -65,7 +65,7 @@ bounded-run: check waiting unit= for=memory budget=M slots= age=s bounded-run: check budget=M slots= headroom=M starts= ``` -The first line gives the slots of the queue and their size, the slots that are free now, the free memory, the unused budgets of the running commands, the processors of the machine, the processors that the running bounded commands can use together (`cpus-used`), the processors that the check keeps for the programs outside the queue (`cpus-headroom`, a tenth of `cpus`, rounded up), and the processors that are left for a new bounded command (`cpus-free`: `cpus` less `cpus-headroom` and `cpus-used`, and at least 0). `line=busy` means that a command waits in the line. `fits` is the largest budget whose memory, with its own headroom, is free now. It does not count the slots or the line; `starts` counts all three. Each `held` line is one bounded command that runs now: its budget, the memory it uses, the processors it can use, how long it has run, its working directory, and its command. The held lines, `cpus-used`, and `cpus-free` need the hard cap, so without it there are no held lines and `cpus-used` and `cpus-free` are `-`. Each `waiting` line is one bounded command that had its turn in the queue and now waits for free memory: its budget, the slots that it holds, how long it has waited, its working directory, and its command. It keeps its slots while it waits, so it does not lose its turn to a command behind it in the line. A `waiting` line means that the machine is short of memory, not of slots: the programs outside the queue hold most of the memory, and the command waits for the memory that a running bounded command of a `held` line frees when it ends. It starts when that memory is free, or exits 75 when no bounded command runs any more. Each `budget` line is one `--memory`, in the order given: the slots that it needs, the headroom that the wait keeps free beside that budget, and `starts=yes` when a command of that budget would start now. A `-` means that the check cannot read or count the value at that moment. +The first line gives the slots of the queue and their size, the slots that are free now, the free memory, the unused budgets of the running commands, the processors of the machine, the processors that the running bounded commands can use together (`cpus-used`), the processors that the check keeps for the programs outside the queue (`cpus-headroom`, a tenth of `cpus`, rounded up), and the processors that are left for a new bounded command (`cpus-free`: `cpus` less `cpus-headroom` and `cpus-used`, and at least 0). `line=busy` means that a command waits in the line. `fits` is the largest budget whose memory, with its own headroom, is free now. It does not count the slots or the line; `starts` counts all three. Each `held` line is one bounded command that runs now: its budget, the memory it uses, the processors it can use, how long it has run, its working directory, and its command. The held lines, `cpus-used`, and `cpus-free` need the hard cap, so without it there are no held lines and `cpus-used` and `cpus-free` are `-`. Each `waiting` line is one bounded command that had its turn in the queue and now waits for free memory: its budget, the slots that it holds, how long it has waited, its working directory, and its command. It keeps its slots while it waits, so it does not lose its turn to a command behind it in the line. A `waiting` line means that the machine is short of memory, not of slots: the programs outside the queue hold most of the memory, and the command waits for the memory that a running bounded command frees when it ends. Where the hard cap is available, that command has a `held` line. It starts when that memory is free, or exits 75 when no bounded command runs any more. Each `budget` line is one `--memory`, in the order given: the slots that it needs, the headroom that the wait keeps free beside that budget, and `starts=yes` when a command of that budget would start now. A `-` means that the check cannot read or count the value at that moment. The exit code of the wrapper is the command's exit code. Exit code 125, with a `bounded-run:` line that names a fault and with no line for the budget and the peak, means that the wrapper itself failed (see §When the wrapper cannot run). @@ -159,7 +159,7 @@ A command that the project adds later gets its entry when an agent first needs i ## Rules for the wait -A long wait means that the machine is full. It is never a fault to repair: the queue has no state that can go wrong between runs, and an agent that repairs it breaks it. Waiting commands take turns in a line. Only the command at the head of the line tries for slots, so the smaller commands behind it do not take the slots that it waits for. The line has no fixed order, and a head that sees no slot come free for 5 minutes lets the commands behind it try first. So a small command can wait behind a large one, also when a slot is free. Unless an instruction says otherwise (§When an instruction sends a command around the wrapper), the only correct moves are to keep the run in the queue and to tell the user that the machine is full and the command is queued. After a long wait, run the wrapper with `--check`, and in that message name each command of its `held` and `waiting` lines, with its directory and its age, so that the user can see what holds the machine. When the check has `waiting` lines, also say that the machine is short of memory, not of slots: the programs outside the queue hold most of the memory, and the waiting commands start when the `held` commands end. A subagent tells its dispatcher in its report. +A long wait means that the machine is full. It is never a fault to repair: the queue has no state that can go wrong between runs, and an agent that repairs it breaks it. Waiting commands take turns in a line. Only the command at the head of the line tries for slots, so the smaller commands behind it do not take the slots that it waits for. The line has no fixed order, and a head that sees no slot come free for 5 minutes lets the commands behind it try first. So a small command can wait behind a large one, also when a slot is free. Unless an instruction says otherwise (§When an instruction sends a command around the wrapper), the only correct moves are to keep the run in the queue and to tell the user that the machine is full and the command is queued. After a long wait, run the wrapper with `--check`, and in that message name each command of its `held` and `waiting` lines, with its directory and its age, so that the user can see what holds the machine. When the check has `waiting` lines, also say that the machine is short of memory, not of slots: the programs outside the queue hold most of the memory, and the waiting commands start when a running bounded command ends and frees its memory. A subagent tells its dispatcher in its report. - **Run a bounded command in the background, with no short timeout around it.** It can wait in the queue for minutes before it starts. Where the harness requires a timeout, use the longest one that it permits. When that timeout ends a run, start the run again through the wrapper, and say in your report that the timeout ended the run and that the code did not fail. A timeout that you calculate from the run time of the command does not count the wait. A timeout that fires during the wait kills a command that never ran, and the result reads like a failure of the code. - **Plan the time as if the commands run one after the other.** Two bounded commands that start together are safe, because the queue puts them in sequence when the machine cannot hold both. So a parallel start does not always save time, and a promise to the user that depends on it is a guess.