From 826feb0646857de5d9d329167bf0f95cda9fda14 Mon Sep 17 00:00:00 2001 From: Damon Oehlman Date: Mon, 26 Oct 2026 11:21:52 +1000 Subject: [PATCH 1/4] . --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index b64a14a..eba07bf 100644 --- a/flake.lock +++ b/flake.lock @@ -99,11 +99,11 @@ "nixpkgs": "nixpkgs_18" }, "locked": { - "lastModified": 1792919588, - "narHash": "sha256-ROu9Sa8cJ51BsTYrnovgWFeP0gR56wTaI4FPrqVzGOE=", + "lastModified": 1792925781, + "narHash": "sha256-yVIHTOWaqM7njyYMyH+ezKlTVaZUP+jEmt+2wd1TIV4=", "owner": "defrag-au", "repo": "defrag-nix", - "rev": "67468f0c99d5f60e355d616232a342883db67162", + "rev": "be7ddfb3a0015fd5667f57e1cb1c678e729cd33a", "type": "github" }, "original": { From 9831e39d7e55d7ce2595d16e0557d59463401725 Mon Sep 17 00:00:00 2001 From: Damon Oehlman Date: Mon, 26 Oct 2026 12:10:11 +1000 Subject: [PATCH 2/4] tui implementation --- Cargo.lock | 642 ++++++++++++++++++++++++++++++++- Cargo.toml | 9 +- README.md | 1 + crates/cli/Cargo.toml | 1 + crates/cli/src/cli.rs | 59 ++- crates/cli/tests/up.rs | 6 + crates/stack/src/ports.rs | 27 ++ crates/supervisor/src/cache.rs | 44 +++ crates/supervisor/src/run.rs | 33 +- crates/top/Cargo.toml | 15 + crates/top/src/app.rs | 266 ++++++++++++++ crates/top/src/lib.rs | 116 ++++++ crates/top/src/model.rs | 484 +++++++++++++++++++++++++ crates/top/src/poll.rs | 162 +++++++++ crates/top/src/ui.rs | 277 ++++++++++++++ docs/01-model.md | 5 +- docs/03-cli.md | 44 +++ 17 files changed, 2166 insertions(+), 25 deletions(-) create mode 100644 crates/top/Cargo.toml create mode 100644 crates/top/src/app.rs create mode 100644 crates/top/src/lib.rs create mode 100644 crates/top/src/model.rs create mode 100644 crates/top/src/poll.rs create mode 100644 crates/top/src/ui.rs diff --git a/Cargo.lock b/Cargo.lock index 97a2d7a..f643a72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "1.0.0" @@ -52,6 +58,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "assert_cmd" version = "2.2.2" @@ -67,6 +82,12 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "bitflags" version = "2.13.2" @@ -93,6 +114,21 @@ dependencies = [ "serde_core", ] +[[package]] +name = "by_address" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64fa3c856b712db6612c019f14756e64e4bcea13337a6b33b696333a9eaa2d06" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -130,7 +166,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.9", ] [[package]] @@ -145,6 +181,20 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + [[package]] name = "container-compose-cli" version = "0.1.0" @@ -155,6 +205,7 @@ dependencies = [ "container-compose-runtime", "container-compose-stack", "container-compose-supervisor", + "container-compose-top", "serde", "serde_json", "tempfile", @@ -204,6 +255,26 @@ dependencies = [ "tempfile", ] +[[package]] +name = "container-compose-top" +version = "0.1.0" +dependencies = [ + "container-compose-runtime", + "container-compose-stack", + "container-compose-supervisor", + "ratatui", + "serde_json", +] + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -213,6 +284,33 @@ dependencies = [ "libc", ] +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -223,6 +321,68 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + [[package]] name = "difflib" version = "0.4.0" @@ -239,12 +399,27 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dotenvy" version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + [[package]] name = "equivalent" version = "1.0.3" @@ -276,6 +451,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "generic-array" version = "0.14.7" @@ -302,6 +483,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -309,6 +495,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "indexmap" version = "2.14.2" @@ -319,36 +511,159 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "instability" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3b5acc1e2fd9375041a388da33d1eb8aed5f7a8c0dd3543e3ea2805adfbe20" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "kasuari" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f14c90e40a6133f4805b83542a7ed7364f56070b447f56e7f4e90297122209" +dependencies = [ + "hashbrown", + "portable-atomic", + "thiserror", +] + [[package]] name = "libc" version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "line-clipping" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e752191d037c44ad111a8caa762921926658402f01cc1253f7bef2020ece4f5e" +dependencies = [ + "bitflags", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litrs" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4744e383959f0db86ede514b809b1c53251889093803c05267acc7d4e7030d70" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown", +] + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabcc654086861f9e1bc5115258de62ef92c79b20f4615a1b0cc57c4e82027d4" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -361,6 +676,74 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "palette" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddeed8580d347d2abf3dcf06a5f0b3dc020258338526b277847cd4248a70fc64" +dependencies = [ + "approx", + "libm", + "palette_derive", + "palette_math", +] + +[[package]] +name = "palette_derive" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88537020289b719d81be994ccf1bbf4990f477e2f69ee52fe3e45f43a02e56be" +dependencies = [ + "by_address", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "palette_math" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e6eb142958d64335fb0e345c5b9ead2ecd6fc438c307e9d7d3c4fd428dbaf12" +dependencies = [ + "libm", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "predicates" version = "3.1.4" @@ -412,12 +795,96 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "ratatui" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3274ba0a2c5e1bcad2a2005d20f4dc59dad26b2eb0940fb094500dba4099d57d" +dependencies = [ + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-widgets", + "serde", +] + +[[package]] +name = "ratatui-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb175c433c8e28a809d1f5773a2ae96e68c0ce40db865cbab1020bf33ae479c" +dependencies = [ + "bitflags", + "compact_str", + "hashbrown", + "itertools", + "kasuari", + "lru", + "palette", + "serde", + "strum", + "thiserror", + "unicode-segmentation", + "unicode-truncate", + "unicode-width", +] + +[[package]] +name = "ratatui-crossterm" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567584a3b0e6a8203c23de40b4861497266725eb5363dbfd18a1edd603cca9f0" +dependencies = [ + "cfg-if", + "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e3d19bcc9130ca376277d93b60767ff121ace3be06f5f95f81dd68956407d1" +dependencies = [ + "bitflags", + "hashbrown", + "indoc", + "instability", + "itertools", + "line-clipping", + "ratatui-core", + "serde", + "strum", + "time", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regex-automata" version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.5" @@ -431,6 +898,30 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.229" @@ -458,7 +949,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.9", ] [[package]] @@ -515,6 +1006,17 @@ dependencies = [ "signal-hook-registry", ] +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -525,12 +1027,56 @@ dependencies = [ "libc", ] +[[package]] +name = "smallvec" +version = "1.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.9" @@ -561,6 +1107,47 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + [[package]] name = "toml" version = "0.8.23" @@ -614,6 +1201,29 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-truncate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" +dependencies = [ + "itertools", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "utf8parse" version = "0.2.2" @@ -635,6 +1245,34 @@ dependencies = [ "libc", ] +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 1cf6692..b6a1c2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/runtime", "crates/plan", "crates/supervisor", + "crates/top", "crates/cli", "dev/fake-container", ] @@ -21,11 +22,17 @@ container-compose-plan = { path = "crates/plan" } container-compose-runtime = { path = "crates/runtime" } container-compose-stack = { path = "crates/stack" } container-compose-supervisor = { path = "crates/supervisor" } +container-compose-top = { path = "crates/top" } assert_cmd = "2" clap = { version = "4", features = ["derive"] } dotenvy = "0.15" -serde = { version = "1", features = ["derive"] } +# `top` only. The crossterm backend alone: the calendar widget, the macros and +# the layout cache are default features nothing here uses. Ratatui re-exports +# the crossterm it was built against as `ratatui::crossterm`; there is no +# separate crossterm pin, so two versions cannot end up in the tree. +ratatui = { version = "0.30.2", default-features = false, features = ["crossterm"] } +serde ={ version = "1", features = ["derive"] } serde_json = "1" serde_path_to_error = "0.1" sha2 = "0.10" diff --git a/README.md b/README.md index 60e8393..fb9d0a6 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ domain was registered do not get a name — `down` then `up` after setting it. | `up` | create and start in dependency order, then supervise until Ctrl-C | | `down` | stop and delete what this project owns; `--volumes`, `--timeout` | | `ps` | what is running, with health; `--probe`, `--json` | +| `top` | `ps` kept live in the terminal, with restarts, changes and the selected service's logs | | `logs`, `exec`, `run` | the runtime's own, scoped to the stack | | `build`, `pull` | the images the stack declares | diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index d9798b2..feef2ad 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -15,6 +15,7 @@ container-compose-plan.workspace = true container-compose-runtime.workspace = true container-compose-stack.workspace = true container-compose-supervisor.workspace = true +container-compose-top.workspace = true serde.workspace = true serde_json.workspace = true toml.workspace = true diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index 394c5b3..a362871 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -1,7 +1,7 @@ //! The command line. //! -//! `config`, `plan`, `up` (foreground and detached), `down`, `ps` and `logs` -//! are implemented. The foreground `up` is the supervisor. +//! `config`, `plan`, `up` (foreground and detached), `down`, `ps`, `top` and +//! `logs` are implemented. The foreground `up` is the supervisor. use std::collections::BTreeMap; use std::io::IsTerminal; @@ -19,7 +19,6 @@ use container_compose_runtime::{ use container_compose_stack::duration::Span; use container_compose_stack::env::{self, Guard}; use container_compose_stack::model::{PullPolicy, RestartPolicy}; -use container_compose_stack::ports::Protocol; use container_compose_stack::resolve::{ MountSource, ResolvedHealthcheck, ResolvedService, ResolvedStack, }; @@ -184,6 +183,15 @@ pub enum Command { #[arg(long)] dry_run: bool, }, + /// A live, read-only dashboard: state, health, restarts and logs per service. + Top { + /// How often the runtime is polled. + #[arg(long, default_value = "1s", value_parser = parse_span)] + interval: Span, + /// Log lines fetched for the selected service. + #[arg(short = 'n', long, default_value = "200")] + lines: u32, + }, /// Print a service's logs. Logs { /// Services to show; default is all. @@ -229,6 +237,7 @@ pub fn run(cli: Cli) -> Result<(), Error> { dry_run, } => down_command(&cli, *volumes, *timeout, *dry_run), Command::Ps { json, probe } => ps_command(&cli, *json, *probe), + Command::Top { interval, lines } => top_command(&cli, *interval, *lines), Command::Logs { services, follow, @@ -538,6 +547,34 @@ fn ps_command(cli: &Cli, json: bool, probe_now: bool) -> Result<(), Error> { Ok(()) } +fn top_command(cli: &Cli, interval: Span, lines: u32) -> Result<(), Error> { + // Like `ps`, `top` works without the stack file, so an interrupted session + // stays inspectable. A file that exists but does not load is still an error. + let stack = if cli.file.exists() { + Some(stack::load( + &cli.file, + &Options { + project_name: cli.project_name.clone(), + }, + )?) + } else { + None + }; + let project = stack + .as_ref() + .map(|stack| stack.name.clone()) + .unwrap_or_else(|| project_from_path(cli)); + + container_compose_top::run(container_compose_top::Options { + runtime: cli.runtime.clone(), + project, + stack, + interval: interval.as_duration(), + log_lines: lines, + }) + .map_err(|error| Error::Runtime(error.to_string())) +} + struct TerminalFlags { interactive: bool, tty: bool, @@ -1205,21 +1242,7 @@ fn push_ps_row(out: &mut String, cells: &[String; 6], widths: &[usize; 6]) { } fn render_ports(service: &ResolvedService) -> Vec { - service - .ports - .iter() - .map(|port| { - let mut rendered = String::new(); - if let Some((address, host_port)) = port.host_binding() { - rendered.push_str(&format!("{address}:{host_port}->")); - } - rendered.push_str(&port.container_port.to_string()); - if port.protocol == Protocol::Udp { - rendered.push_str("/udp"); - } - rendered - }) - .collect() + service.ports.iter().map(|port| port.display()).collect() } /// Known services in start order, then anything else by name. diff --git a/crates/cli/tests/up.rs b/crates/cli/tests/up.rs index 2d3a948..274c31a 100644 --- a/crates/cli/tests/up.rs +++ b/crates/cli/tests/up.rs @@ -1116,6 +1116,9 @@ fn ps_reads_a_live_supervisors_health_cache() { CachedHealth { state: "healthy".to_owned(), checked_at: Some("2026-10-24T00:00:01Z".to_owned()), + restarts: 0, + last_restart_at: None, + gave_up: false, }, )]), }, @@ -1142,6 +1145,9 @@ fn ps_reads_a_live_supervisors_health_cache() { CachedHealth { state: "healthy".to_owned(), checked_at: None, + restarts: 0, + last_restart_at: None, + gave_up: false, }, )]), }, diff --git a/crates/stack/src/ports.rs b/crates/stack/src/ports.rs index de04fa9..0437f82 100644 --- a/crates/stack/src/ports.rs +++ b/crates/stack/src/ports.rs @@ -123,6 +123,20 @@ impl PortSpec { .unwrap_or_else(|| "127.0.0.1".to_owned()); Some((ip, port)) } + + /// The `ps` form: `127.0.0.1:8184->8184`, `8184` when nothing is published, + /// with `/udp` appended for UDP. + pub fn display(&self) -> String { + let mut rendered = String::new(); + if let Some((address, host_port)) = self.host_binding() { + rendered.push_str(&format!("{address}:{host_port}->")); + } + rendered.push_str(&self.container_port.to_string()); + if self.protocol == Protocol::Udp { + rendered.push_str("/udp"); + } + rendered + } } fn parse_port(value: &str, whole: &str) -> Result { @@ -155,6 +169,19 @@ mod tests { assert_eq!(loopback.host_port, Some(8184)); } + #[test] + fn displays_in_the_ps_form() { + assert_eq!( + PortSpec::parse("127.0.0.1:9944:9944").unwrap().display(), + "127.0.0.1:9944->9944" + ); + assert_eq!(PortSpec::parse("8184").unwrap().display(), "8184"); + assert_eq!( + PortSpec::parse("127.0.0.1:5353:53/udp").unwrap().display(), + "127.0.0.1:5353->53/udp" + ); + } + #[test] fn parses_bracketed_ipv6() { let spec = PortSpec::parse("[::1]:8080:8000/udp").unwrap(); diff --git a/crates/supervisor/src/cache.rs b/crates/supervisor/src/cache.rs index 854fcd8..7e05866 100644 --- a/crates/supervisor/src/cache.rs +++ b/crates/supervisor/src/cache.rs @@ -31,6 +31,17 @@ pub struct HealthCache { pub struct CachedHealth { pub state: String, pub checked_at: Option, + /// Restarts this supervisor has issued for the service since it started. + /// Unlike the tick's attempt counter, this never resets on stable uptime: + /// it is history for a reader, not backoff state. Absent in a cache + /// written before it existed, which reads as zero. + #[serde(default)] + pub restarts: u32, + #[serde(default)] + pub last_restart_at: Option, + /// The supervisor stopped restarting the service (`max_retries` reached). + #[serde(default)] + pub gave_up: bool, } pub fn dir() -> Option { @@ -136,11 +147,44 @@ mod tests { CachedHealth { state: "healthy".to_owned(), checked_at: Some("2026-10-24T00:00:01Z".to_owned()), + restarts: 0, + last_restart_at: None, + gave_up: false, }, )]), } } + #[test] + fn a_cache_written_before_restart_history_still_reads() { + // The document this crate wrote before `restarts`, `last_restart_at` + // and `gave_up` existed, as captured from a live midnight-lab + // supervisor's cache file. + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("midnight-lab.json"); + std::fs::write( + &path, + r#"{ + "pid": 43564, + "project": "midnight-lab", + "config": "2a9ac5706fb80bbd", + "updated_at": "2026-10-26T01:18:08Z", + "services": { + "indexer": { "state": "unknown", "checked_at": null }, + "node": { "state": "healthy", "checked_at": "2026-10-26T01:18:08Z" } + } +}"#, + ) + .unwrap(); + + let read_back = read_at(&path).expect("an older cache still parses"); + let node = &read_back.services["node"]; + assert_eq!(node.state, "healthy"); + assert_eq!(node.restarts, 0); + assert_eq!(node.last_restart_at, None); + assert!(!node.gave_up); + } + #[test] fn a_cache_round_trips_and_leaves_no_temporary_behind() { let dir = tempfile::TempDir::new().unwrap(); diff --git a/crates/supervisor/src/run.rs b/crates/supervisor/src/run.rs index 6e2b144..17ba8d5 100644 --- a/crates/supervisor/src/run.rs +++ b/crates/supervisor/src/run.rs @@ -58,6 +58,15 @@ impl From for SupervisorError { } } +/// What this supervisor has done to one service, for readers of the cache. +/// Owned by the loop, so the tick's backoff state stays the tick's. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct History { + restarts: u32, + last_restart_at: Option, + gave_up: bool, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum OnInterrupt { Leave, @@ -94,6 +103,7 @@ pub fn run( .collect(), ); let start = Instant::now(); + let mut history: BTreeMap = BTreeMap::new(); writeln!( out, @@ -101,7 +111,7 @@ pub fn run( services.len(), if services.len() == 1 { "" } else { "s" } )?; - publish(stack, &supervisor, out); + publish(stack, &supervisor, &history, out); loop { if interrupted(&interrupts) { @@ -147,10 +157,18 @@ pub fn run( "give up {service} after {attempts} restart{}", if attempts == 1 { "" } else { "s" } ); + history.entry(service).or_default().gave_up = true; + cache_dirty = true; } Action::Restart { service, attempt } => { let name = argv::container_name(&stack.name, &service); let _ = writeln!(out, "restart {name} attempt {attempt}"); + let record = history.entry(service).or_default(); + record.restarts = record.restarts.saturating_add(1); + record.last_restart_at = + Some(clock::format_rfc3339_seconds(clock::wall_ms() / 1_000)); + record.gave_up = false; + cache_dirty = true; if let Err(error) = Command::new(&options.runtime) .args(argv::start_container(&name)) .run() @@ -182,7 +200,7 @@ pub fn run( } if cache_dirty { - publish(stack, &supervisor, out); + publish(stack, &supervisor, &history, out); } sleep(options.poll_interval, &interrupts); } @@ -260,7 +278,12 @@ fn views_for( .collect() } -fn publish(stack: &ResolvedStack, supervisor: &Supervisor, out: &mut dyn Write) { +fn publish( + stack: &ResolvedStack, + supervisor: &Supervisor, + history: &BTreeMap, + out: &mut dyn Write, +) { let services = supervisor .health_snapshot() .into_iter() @@ -268,11 +291,15 @@ fn publish(stack: &ResolvedStack, supervisor: &Supervisor, out: &mut dyn Write) let checked_at = supervisor .checked_at(&name) .map(|millis| clock::format_rfc3339_seconds(millis / 1_000)); + let record = history.get(&name).cloned().unwrap_or_default(); ( name, CachedHealth { state: health.as_str().to_owned(), checked_at, + restarts: record.restarts, + last_restart_at: record.last_restart_at, + gave_up: record.gave_up, }, ) }) diff --git a/crates/top/Cargo.toml b/crates/top/Cargo.toml new file mode 100644 index 0000000..4ce0655 --- /dev/null +++ b/crates/top/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "container-compose-top" +version.workspace = true +edition.workspace = true +publish = false +description = "A read-only terminal dashboard for a running stack" + +[dependencies] +container-compose-runtime.workspace = true +container-compose-stack.workspace = true +container-compose-supervisor.workspace = true +ratatui.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/crates/top/src/app.rs b/crates/top/src/app.rs new file mode 100644 index 0000000..8267d59 --- /dev/null +++ b/crates/top/src/app.rs @@ -0,0 +1,266 @@ +//! The dashboard's state, and how a poll or a key changes it. +//! +//! No terminal here: [`App::apply`] takes an [`Update`] and [`App::on_key`] +//! a key, and each returns the [`Request`] the poller should receive, if any. + +use std::collections::VecDeque; + +use ratatui::crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; +use ratatui::widgets::TableState; + +use crate::model::{self, Declared, Event, Row}; +use crate::poll::{Logs, Request, Snapshot, Update}; + +/// Events kept for the events pane; older ones fall off. +const EVENT_HISTORY: usize = 200; + +#[derive(Debug)] +pub struct App { + pub declared: Declared, + pub rows: Vec, + pub table: TableState, + /// Newest first. + pub events: VecDeque, + pub logs: Option, + /// Lines scrolled up from the end of the log; 0 follows the tail. + pub log_scroll: usize, + pub runtime_error: Option, + pub supervisor_pid: Option, + pub updated_at: Option, + /// The last on-demand probe, shown in the footer. + pub note: Option, + pub quit: bool, + polled: bool, +} + +impl App { + pub fn new(declared: Declared) -> Self { + Self { + declared, + rows: Vec::new(), + table: TableState::default(), + events: VecDeque::new(), + logs: None, + log_scroll: 0, + runtime_error: None, + supervisor_pid: None, + updated_at: None, + note: None, + quit: false, + polled: false, + } + } + + pub fn selected_service(&self) -> Option<&str> { + self.table + .selected() + .and_then(|index| self.rows.get(index)) + .map(|row| row.service.as_str()) + } + + pub fn apply(&mut self, update: Update) -> Option { + match update { + Update::Snapshot(snapshot) => self.apply_snapshot(snapshot), + Update::Probed { service, outcome } => { + self.note = Some(format!("probe {service}: {outcome}")); + None + } + } + } + + fn apply_snapshot(&mut self, snapshot: Snapshot) -> Option { + let before = self.selected_service().map(str::to_owned); + self.updated_at = Some(model::clock_of_day(snapshot.at_secs)); + self.supervisor_pid = snapshot.cache.as_ref().map(|cache| cache.pid); + + match snapshot.containers { + Ok(containers) => { + self.runtime_error = None; + let rows = model::rows( + &self.declared, + &containers, + snapshot.cache.as_ref(), + snapshot.at_secs, + ); + // The first poll is the baseline, not a burst of "appeared". + if self.polled { + for event in model::events(&self.rows, &rows, snapshot.at_secs) { + self.events.push_front(event); + } + self.events.truncate(EVENT_HISTORY); + } + self.polled = true; + self.rows = rows; + } + // Keep the last rows on screen; the header says they are stale. + Err(error) => self.runtime_error = Some(error), + } + + self.keep_selection(before.as_deref()); + let after = self.selected_service().map(str::to_owned); + + // A poll that raced a selection change carries the previous service's + // lines; only the selected service's logs are shown. + self.logs = snapshot + .logs + .filter(|logs| after.as_deref() == Some(logs.service.as_str())); + + (after != before).then_some(Request::Select(after)) + } + + /// Stay on the same service when rows move; fall back to the first row. + fn keep_selection(&mut self, service: Option<&str>) { + let index = service + .and_then(|service| self.rows.iter().position(|row| row.service == service)) + .or_else(|| (!self.rows.is_empty()).then_some(0)); + self.table.select(index); + } + + pub fn on_key(&mut self, key: KeyEvent) -> Option { + match key.code { + KeyCode::Char('q') | KeyCode::Esc => { + self.quit = true; + None + } + KeyCode::Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) => { + self.quit = true; + None + } + KeyCode::Down | KeyCode::Char('j') => self.move_selection(1), + KeyCode::Up | KeyCode::Char('k') => self.move_selection(-1), + KeyCode::PageUp => { + self.log_scroll = self.log_scroll.saturating_add(10); + None + } + KeyCode::PageDown => { + self.log_scroll = self.log_scroll.saturating_sub(10); + None + } + KeyCode::End => { + self.log_scroll = 0; + None + } + KeyCode::Char('r') => Some(Request::Refresh), + KeyCode::Char('p') => { + let service = self.selected_service()?.to_owned(); + self.note = Some(format!("probing {service}…")); + Some(Request::Probe(service)) + } + _ => None, + } + } + + fn move_selection(&mut self, step: isize) -> Option { + if self.rows.is_empty() { + return None; + } + let last = self.rows.len() - 1; + let current = self.table.selected().unwrap_or(0); + let next = current.saturating_add_signed(step).min(last); + if next == current { + return None; + } + self.table.select(Some(next)); + self.log_scroll = 0; + // The old service's lines would sit under the new service's title. + self.logs = None; + Some(Request::Select(self.selected_service().map(str::to_owned))) + } +} + +#[cfg(test)] +mod tests { + use super::App; + use crate::model::Declared; + use crate::poll::{Logs, Request, Snapshot, Update}; + use container_compose_runtime::ObservedContainer; + use ratatui::crossterm::event::{KeyCode, KeyEvent}; + + fn captured() -> Vec { + let text = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../fixtures/real/container-list.json" + )) + .unwrap(); + serde_json::from_str(&text).unwrap() + } + + fn app() -> App { + let project = captured()[0].project().unwrap().to_owned(); + App::new(Declared::bare(project)) + } + + fn snapshot(containers: Vec, logs: Option) -> Update { + Update::Snapshot(Snapshot { + at_secs: 0, + containers: Ok(containers), + cache: None, + logs, + }) + } + + #[test] + fn the_first_poll_selects_the_first_row_and_asks_for_its_logs() { + let mut app = app(); + let request = app.apply(snapshot(captured(), None)); + assert_eq!(request, Some(Request::Select(Some("api".to_owned())))); + assert!(app.events.is_empty(), "the first poll is the baseline"); + } + + #[test] + fn moving_down_selects_the_next_service_and_clears_its_logs() { + let mut app = app(); + app.apply(snapshot( + captured(), + Some(Logs { + service: "api".to_owned(), + lines: vec!["capture-hello".to_owned()], + error: None, + }), + )); + let request = app.on_key(KeyEvent::from(KeyCode::Down)); + assert_eq!(request, Some(Request::Select(Some("worker".to_owned())))); + assert!(app.logs.is_none()); + + // Already on the last row: nothing to ask for. + assert_eq!(app.on_key(KeyEvent::from(KeyCode::Down)), None); + } + + #[test] + fn a_runtime_error_keeps_the_last_rows() { + let mut app = app(); + app.apply(snapshot(captured(), None)); + app.apply(Update::Snapshot(Snapshot { + at_secs: 1, + containers: Err("`container list` exited with 1".to_owned()), + cache: None, + logs: None, + })); + assert_eq!(app.rows.len(), 2); + assert!(app.runtime_error.is_some()); + } + + #[test] + fn a_container_that_disappears_is_an_event() { + let mut app = app(); + app.apply(snapshot(captured(), None)); + let only_api: Vec = captured() + .into_iter() + .filter(|container| container.service() == Some("api")) + .collect(); + app.apply(snapshot(only_api, None)); + let texts: Vec = app + .events + .iter() + .map(|event| format!("{} {}", event.service, event.text)) + .collect(); + assert_eq!(texts, ["worker removed"]); + } + + #[test] + fn q_quits() { + let mut app = app(); + app.on_key(KeyEvent::from(KeyCode::Char('q'))); + assert!(app.quit); + } +} diff --git a/crates/top/src/lib.rs b/crates/top/src/lib.rs new file mode 100644 index 0000000..bf26ce2 --- /dev/null +++ b/crates/top/src/lib.rs @@ -0,0 +1,116 @@ +//! `container-compose top`: a read-only terminal dashboard for a stack. +//! +//! It shows what `ps` shows, kept current — state, health, uptime, restarts +//! and ports per service — plus what changed between polls and the selected +//! service's logs. It reads the same two sources `ps` does: the runtime's +//! container list and the supervisor's health cache. It takes no lock and +//! starts or stops nothing, so it runs beside a supervisor, not instead of one. +//! +//! The split, so the logic is testable without a terminal: +//! +//! - [`model`] is pure: a poll and the clock in, rows and events out. +//! - [`poll`] is the worker thread that owns every runtime call. +//! - [`app`] is the state a poll or a key changes. +//! - [`ui`] draws it. + +pub mod app; +pub mod model; +pub mod poll; +pub mod ui; + +use std::collections::BTreeMap; +use std::io::{self, IsTerminal}; +use std::time::Duration; + +use ratatui::crossterm::event; +use ratatui::DefaultTerminal; + +use container_compose_stack::resolve::ResolvedStack; + +use crate::app::App; +use crate::model::Declared; +use crate::poll::{Request, Source}; + +/// How long a key wait lasts before the screen redraws with whatever the +/// poller sent meanwhile. +const FRAME: Duration = Duration::from_millis(200); + +#[derive(Debug, Clone)] +pub struct Options { + pub runtime: String, + /// The project to show; from the stack file, or the directory as `ps` does. + pub project: String, + /// The stack, when its file exists. Without it, rows come from the + /// project's containers alone and there is no on-demand probe. + pub stack: Option, + pub interval: Duration, + pub log_lines: u32, +} + +pub fn run(options: Options) -> io::Result<()> { + if !io::stdout().is_terminal() { + return Err(io::Error::other( + "`top` needs a terminal; use `ps --json` from a script", + )); + } + + let declared = options.stack.as_ref().map_or_else( + || Declared::bare(options.project.clone()), + Declared::from_stack, + ); + let healthchecks: BTreeMap, Duration)> = options + .stack + .iter() + .flat_map(|stack| stack.services.iter()) + .filter_map(|(name, service)| { + let check = service.healthcheck.as_ref()?; + Some(( + name.clone(), + (check.exec.clone(), check.timeout.as_duration()), + )) + }) + .collect(); + + let (requests, updates) = poll::spawn(Source { + runtime: options.runtime, + project: declared.project.clone(), + config_hash: declared.config_hash.clone(), + healthchecks, + interval: options.interval, + log_lines: options.log_lines, + }); + let mut app = App::new(declared); + + ratatui::run(|terminal| event_loop(terminal, &mut app, &requests, &updates)) +} + +fn event_loop( + terminal: &mut DefaultTerminal, + app: &mut App, + requests: &std::sync::mpsc::Sender, + updates: &std::sync::mpsc::Receiver, +) -> io::Result<()> { + while !app.quit { + while let Ok(update) = updates.try_recv() { + if let Some(request) = app.apply(update) { + send(requests, request)?; + } + } + terminal.draw(|frame| ui::draw(frame, app))?; + + if event::poll(FRAME)? { + if let Some(key) = event::read()?.as_key_press_event() { + if let Some(request) = app.on_key(key) { + send(requests, request)?; + } + } + } + } + Ok(()) +} + +fn send(requests: &std::sync::mpsc::Sender, request: Request) -> io::Result<()> { + requests + .send(request) + .map_err(|_| io::Error::other("the poller stopped")) +} diff --git a/crates/top/src/model.rs b/crates/top/src/model.rs new file mode 100644 index 0000000..fa5ac6b --- /dev/null +++ b/crates/top/src/model.rs @@ -0,0 +1,484 @@ +//! The view model: what one poll of the runtime and the health cache means, +//! as rows a table can draw and events a reader can scan. +//! +//! Pure. Containers, the cache and the clock are parameters, so every rule +//! here is a unit test rather than a terminal session. + +use std::collections::BTreeMap; + +use container_compose_runtime::ObservedContainer; +use container_compose_stack::resolve::ResolvedStack; +use container_compose_supervisor::cache::HealthCache; +use container_compose_supervisor::clock; + +/// What the stack file declares. `top` runs without one, as `ps` does, so an +/// interrupted session stays inspectable; then only the project is known. +#[derive(Debug, Clone, Default)] +pub struct Declared { + pub project: String, + /// Start order. Rows follow it; anything undeclared sorts after, by name. + pub order: Vec, + pub services: BTreeMap, + /// The stack's config hash, so a cache from a different config is ignored. + pub config_hash: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeclaredService { + pub ports: Vec, + pub restart: String, + pub has_healthcheck: bool, +} + +impl Declared { + pub fn from_stack(stack: &ResolvedStack) -> Self { + Self { + project: stack.name.clone(), + order: stack.order.clone(), + services: stack + .services + .iter() + .map(|(name, service)| { + ( + name.clone(), + DeclaredService { + ports: service.ports.iter().map(|port| port.display()).collect(), + restart: service.restart.as_str().to_owned(), + has_healthcheck: service.healthcheck.is_some(), + }, + ) + }) + .collect(), + config_hash: Some(stack.config_hash.clone()), + } + } + + pub fn bare(project: String) -> Self { + Self { + project, + ..Self::default() + } + } +} + +/// The state shown for a declared service that has no container. +pub const NOT_CREATED: &str = "not created"; +/// Health of a service that is not running: there is nothing to be healthy. +pub const NO_HEALTH: &str = "—"; +/// Health of a running service that declares no healthcheck. +pub const NO_CHECK: &str = "no check"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row { + pub service: String, + pub container: Option, + /// The runtime's word (`running`, `stopped`), or [`NOT_CREATED`]. + pub state: String, + pub health: String, + pub uptime: Option, + pub restarts: u32, + pub gave_up: bool, + pub policy: Option, + pub ports: Vec, + pub image: Option, +} + +impl Row { + pub fn is_running(&self) -> bool { + self.state == "running" + } +} + +/// One row per declared service, in start order, then one per undeclared +/// container of the project. Only a cache that [`cache::live`] trusted +/// should be passed: health and restarts come from it alone. +/// +/// [`cache::live`]: container_compose_supervisor::cache::live +pub fn rows( + declared: &Declared, + containers: &[ObservedContainer], + cache: Option<&HealthCache>, + now_secs: u64, +) -> Vec { + let mine: Vec<&ObservedContainer> = containers + .iter() + .filter(|container| container.project() == Some(declared.project.as_str())) + .collect(); + + let mut rows: Vec = declared + .order + .iter() + .map(|service| { + let container = mine + .iter() + .copied() + .find(|container| container.service() == Some(service.as_str())); + row( + service, + container, + declared.services.get(service), + cache, + now_secs, + ) + }) + .collect(); + + let mut undeclared: Vec = mine + .iter() + .copied() + .filter(|container| { + container + .service() + .is_none_or(|service| !declared.order.iter().any(|known| known == service)) + }) + .map(|container| { + let service = container.service().unwrap_or(&container.id).to_owned(); + row(&service, Some(container), None, cache, now_secs) + }) + .collect(); + undeclared.sort_by(|a, b| a.service.cmp(&b.service)); + rows.extend(undeclared); + rows +} + +fn row( + service: &str, + container: Option<&ObservedContainer>, + declared: Option<&DeclaredService>, + cache: Option<&HealthCache>, + now_secs: u64, +) -> Row { + let cached = cache.and_then(|cache| cache.services.get(service)); + let running = container.is_some_and(ObservedContainer::is_running); + + let health = if !running { + NO_HEALTH.to_owned() + } else if declared.is_some_and(|service| !service.has_healthcheck) { + NO_CHECK.to_owned() + } else { + cached.map_or_else(|| "unknown".to_owned(), |cached| cached.state.clone()) + }; + + let uptime = container + .filter(|container| container.is_running()) + .and_then(|container| container.status.started_date.as_deref()) + .and_then(clock::parse_rfc3339_seconds) + .map(|started| format_span(now_secs.saturating_sub(started))); + + Row { + service: service.to_owned(), + container: container.map(|container| container.id.clone()), + state: container.map_or_else( + || NOT_CREATED.to_owned(), + |container| container.status.state.clone(), + ), + health, + uptime, + restarts: cached.map_or(0, |cached| cached.restarts), + gave_up: cached.is_some_and(|cached| cached.gave_up), + policy: declared.map(|service| service.restart.clone()), + ports: declared + .map(|service| service.ports.clone()) + .unwrap_or_default(), + image: container.map(|container| container.configuration.image.reference.clone()), + } +} + +/// `45s`, `12m 03s`, `5h 07m`, `3d 04h`: two units at most, which is all a +/// glance at uptime needs. +pub fn format_span(seconds: u64) -> String { + let (days, hours, minutes, secs) = ( + seconds / 86_400, + seconds % 86_400 / 3_600, + seconds % 3_600 / 60, + seconds % 60, + ); + if days > 0 { + format!("{days}d {hours:02}h") + } else if hours > 0 { + format!("{hours}h {minutes:02}m") + } else if minutes > 0 { + format!("{minutes}m {secs:02}s") + } else { + format!("{secs}s") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Event { + /// `HH:MM:SSZ`, UTC, the clock the cache and the runtime both use. + pub at: String, + pub service: String, + pub text: String, +} + +/// What changed between two polls, matched by service. Uptime ticking is not +/// an event; a state, a health, a restart or a give-up is. +pub fn events(previous: &[Row], next: &[Row], now_secs: u64) -> Vec { + let at = clock_of_day(now_secs); + let mut events = Vec::new(); + for row in next { + let event = |text: String| Event { + at: at.clone(), + service: row.service.clone(), + text, + }; + let Some(before) = previous.iter().find(|before| before.service == row.service) else { + events.push(event(format!("appeared, {}", row.state))); + continue; + }; + if before.state != row.state { + events.push(event(format!("{} → {}", before.state, row.state))); + } else if before.health != row.health && row.is_running() { + events.push(event(format!("health {} → {}", before.health, row.health))); + } + if row.restarts > before.restarts { + events.push(event(format!("restarted, {} so far", row.restarts))); + } + if row.gave_up && !before.gave_up { + events.push(event("supervisor gave up restarting it".to_owned())); + } + } + for before in previous { + if !next.iter().any(|row| row.service == before.service) { + events.push(Event { + at: at.clone(), + service: before.service.clone(), + text: "removed".to_owned(), + }); + } + } + events +} + +/// `HH:MM:SSZ` from epoch seconds. +pub fn clock_of_day(seconds: u64) -> String { + let stamp = clock::format_rfc3339_seconds(seconds); + // `YYYY-MM-DDTHH:MM:SSZ`; the time of day is everything after the `T`. + stamp + .split_once('T') + .map_or(stamp.clone(), |(_, time)| time.to_owned()) +} + +#[cfg(test)] +mod tests { + use super::{ + events, format_span, rows, Declared, DeclaredService, Row, NOT_CREATED, NO_CHECK, NO_HEALTH, + }; + use container_compose_runtime::ObservedContainer; + use container_compose_supervisor::cache::{CachedHealth, HealthCache}; + use container_compose_supervisor::clock; + use std::collections::BTreeMap; + + /// The `container list --all --format json` response captured from the + /// real runtime for the test suite's fixtures. + fn captured() -> Vec { + let text = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../fixtures/real/container-list.json" + )) + .expect("the captured container list"); + serde_json::from_str(&text).expect("it parses as the observed shape") + } + + /// The captured project and its two services: `api` running, `worker` + /// stopped. `api` declares a healthcheck; `worker` does not. + fn declared(with: &[&str]) -> Declared { + let project = captured()[0].project().unwrap().to_owned(); + let mut order: Vec = vec!["api".to_owned(), "worker".to_owned()]; + order.extend(with.iter().map(|name| (*name).to_owned())); + Declared { + project, + services: order + .iter() + .map(|name| { + ( + name.clone(), + DeclaredService { + ports: Vec::new(), + restart: "no".to_owned(), + has_healthcheck: name == "api", + }, + ) + }) + .collect(), + order, + config_hash: None, + } + } + + fn started_at_of_api() -> u64 { + let api = captured() + .into_iter() + .find(|container| container.service() == Some("api")) + .unwrap(); + clock::parse_rfc3339_seconds(api.status.started_date.as_deref().unwrap()).unwrap() + } + + fn cache(project: &str, api: CachedHealth) -> HealthCache { + HealthCache { + pid: std::process::id(), + project: project.to_owned(), + config: "unused".to_owned(), + updated_at: "2026-10-24T04:26:00Z".to_owned(), + services: BTreeMap::from([("api".to_owned(), api)]), + } + } + + fn healthy(restarts: u32, gave_up: bool) -> CachedHealth { + CachedHealth { + state: "healthy".to_owned(), + checked_at: None, + restarts, + last_restart_at: None, + gave_up, + } + } + + #[test] + fn declared_services_come_first_in_start_order() { + let rows = rows(&declared(&["db"]), &captured(), None, started_at_of_api()); + let services: Vec<&str> = rows.iter().map(|row| row.service.as_str()).collect(); + assert_eq!(services, ["api", "worker", "db"]); + assert_eq!(rows[2].state, NOT_CREATED); + assert_eq!(rows[2].container, None); + } + + #[test] + fn health_comes_from_the_cache_only_while_running() { + let declared = declared(&[]); + let now = started_at_of_api() + 125; + let cache = cache(&declared.project, healthy(2, false)); + let rows = rows(&declared, &captured(), Some(&cache), now); + + assert_eq!(rows[0].state, "running"); + assert_eq!(rows[0].health, "healthy"); + assert_eq!(rows[0].uptime.as_deref(), Some("2m 05s")); + assert_eq!(rows[0].restarts, 2); + + assert_eq!(rows[1].state, "stopped"); + assert_eq!(rows[1].health, NO_HEALTH); + assert_eq!(rows[1].uptime, None); + } + + #[test] + fn a_running_service_without_a_cache_is_unknown_unless_it_has_no_check() { + let declared = declared(&[]); + let rows = rows(&declared, &captured(), None, started_at_of_api()); + assert_eq!(rows[0].health, "unknown"); + + let mut unchecked = declared.clone(); + unchecked.services.get_mut("api").unwrap().has_healthcheck = false; + let rows = super::rows(&unchecked, &captured(), None, started_at_of_api()); + assert_eq!(rows[0].health, NO_CHECK); + } + + #[test] + fn without_a_stack_file_every_project_container_is_a_row() { + let project = captured()[0].project().unwrap().to_owned(); + let rows = rows( + &Declared::bare(project), + &captured(), + None, + started_at_of_api(), + ); + let services: Vec<&str> = rows.iter().map(|row| row.service.as_str()).collect(); + assert_eq!(services, ["api", "worker"]); + assert_eq!(rows[0].policy, None); + } + + #[test] + fn another_projects_containers_are_not_rows() { + let rows = rows( + &Declared::bare("someone-else".to_owned()), + &captured(), + None, + 0, + ); + assert!(rows.is_empty()); + } + + #[test] + fn spans_use_two_units_at_most() { + assert_eq!(format_span(45), "45s"); + assert_eq!(format_span(723), "12m 03s"); + assert_eq!(format_span(18_420), "5h 07m"); + assert_eq!(format_span(273_600), "3d 04h"); + } + + fn base() -> Row { + Row { + service: "indexer".to_owned(), + container: Some("lab-indexer".to_owned()), + state: "running".to_owned(), + health: "starting".to_owned(), + uptime: Some("5s".to_owned()), + restarts: 0, + gave_up: false, + policy: Some("on-stop".to_owned()), + ports: Vec::new(), + image: None, + } + } + + #[test] + fn events_name_state_health_restart_and_give_up_changes() { + let before = base(); + let now = clock::parse_rfc3339_seconds("2026-10-26T01:18:08Z").unwrap(); + + let healthy = Row { + health: "healthy".to_owned(), + uptime: Some("10s".to_owned()), + ..before.clone() + }; + let texts = |next: &Row| -> Vec { + events( + std::slice::from_ref(&before), + std::slice::from_ref(next), + now, + ) + .into_iter() + .map(|event| event.text) + .collect() + }; + assert_eq!(texts(&healthy), ["health starting → healthy"]); + + let stopped = Row { + state: "stopped".to_owned(), + health: NO_HEALTH.to_owned(), + ..before.clone() + }; + assert_eq!(texts(&stopped), ["running → stopped"]); + + let restarted = Row { + restarts: 1, + ..before.clone() + }; + assert_eq!(texts(&restarted), ["restarted, 1 so far"]); + + let abandoned = Row { + state: "stopped".to_owned(), + health: NO_HEALTH.to_owned(), + gave_up: true, + ..before.clone() + }; + assert_eq!( + texts(&abandoned), + ["running → stopped", "supervisor gave up restarting it"] + ); + + let event = &events(std::slice::from_ref(&before), &[healthy], now)[0]; + assert_eq!(event.at, "01:18:08Z"); + assert_eq!(event.service, "indexer"); + } + + #[test] + fn uptime_ticking_is_not_an_event() { + let before = base(); + let later = Row { + uptime: Some("6s".to_owned()), + ..before.clone() + }; + assert!(events(&[before], &[later], 0).is_empty()); + } +} diff --git a/crates/top/src/poll.rs b/crates/top/src/poll.rs new file mode 100644 index 0000000..84d5b5f --- /dev/null +++ b/crates/top/src/poll.rs @@ -0,0 +1,162 @@ +//! The poller: a worker thread that owns every runtime call. +//! +//! `container list` takes a few hundred milliseconds and `container logs` +//! longer on a busy service, so they run here and the UI thread only draws. +//! The UI sends [`Request`]s; the worker sends [`Update`]s back. Dropping the +//! request sender ends the worker. + +use std::collections::BTreeMap; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::thread; +use std::time::{Duration, Instant}; + +use container_compose_runtime::{argv, observe, probe, Command, ObservedContainer, Probe}; +use container_compose_supervisor::cache::{self, HealthCache}; +use container_compose_supervisor::clock; + +/// Where to look, and how often. +#[derive(Debug, Clone)] +pub struct Source { + pub runtime: String, + pub project: String, + pub config_hash: Option, + /// Each declared healthcheck's command and timeout, for an on-demand probe. + pub healthchecks: BTreeMap, Duration)>, + pub interval: Duration, + /// Log lines fetched for the selected service. + pub log_lines: u32, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Request { + /// The service whose logs to fetch; `None` fetches none. Polls at once. + Select(Option), + /// Run the service's healthcheck now, as `ps --probe` does. + Probe(String), + Refresh, +} + +#[derive(Debug)] +pub enum Update { + Snapshot(Snapshot), + Probed { service: String, outcome: String }, +} + +#[derive(Debug)] +pub struct Snapshot { + pub at_secs: u64, + /// The runtime's container list, or why it could not be read. + pub containers: Result, String>, + /// The health cache, only when [`cache::live`] trusts it. + pub cache: Option, + pub logs: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Logs { + pub service: String, + pub lines: Vec, + pub error: Option, +} + +pub fn spawn(source: Source) -> (Sender, Receiver) { + let (requests, inbox) = mpsc::channel(); + let (outbox, updates) = mpsc::channel(); + thread::spawn(move || work(&source, &inbox, &outbox)); + (requests, updates) +} + +fn work(source: &Source, inbox: &Receiver, outbox: &Sender) { + let mut selected: Option = None; + let mut due = Instant::now(); + loop { + match inbox.recv_timeout(due.saturating_duration_since(Instant::now())) { + Ok(Request::Select(service)) => { + selected = service; + due = Instant::now(); + continue; + } + Ok(Request::Refresh) => { + due = Instant::now(); + continue; + } + Ok(Request::Probe(service)) => { + let outcome = probe_now(source, &service); + if outbox.send(Update::Probed { service, outcome }).is_err() { + return; + } + continue; + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => return, + } + + let snapshot = capture(source, selected.as_deref()); + if outbox.send(Update::Snapshot(snapshot)).is_err() { + return; + } + due = Instant::now() + source.interval; + } +} + +fn capture(source: &Source, selected: Option<&str>) -> Snapshot { + // `containers`, not `observe`: it is one subprocess rather than five, and it + // does not gate on `system status`, which answers `unregistered` to a + // launchd session other than the one that started the runtime (an SSH + // login, typically) while `list` works from anywhere. + let containers = observe::containers(&source.runtime).map_err(|error| error.to_string()); + let cache = cache::live(&source.project, source.config_hash.as_deref()); + + let logs = selected.and_then(|service| { + let name = argv::container_name(&source.project, service); + let exists = containers + .as_ref() + .is_ok_and(|list| list.iter().any(|container| container.id == name)); + exists.then(|| fetch_logs(source, service, &name)) + }); + + Snapshot { + at_secs: clock::wall_ms() / 1_000, + containers, + cache, + logs, + } +} + +fn fetch_logs(source: &Source, service: &str, container: &str) -> Logs { + match Command::new(&source.runtime) + .args(argv::logs(container, false, Some(source.log_lines))) + .run() + { + Ok(output) => Logs { + service: service.to_owned(), + lines: output.stdout.lines().map(str::to_owned).collect(), + error: None, + }, + Err(error) => Logs { + service: service.to_owned(), + lines: Vec::new(), + error: Some(error.to_string()), + }, + } +} + +fn probe_now(source: &Source, service: &str) -> String { + let Some((exec, timeout)) = source.healthchecks.get(service) else { + return "declares no healthcheck".to_owned(); + }; + let container = argv::container_name(&source.project, service); + match probe(&source.runtime, &container, exec, *timeout) { + Ok(Probe::Healthy) => "healthy".to_owned(), + Ok(Probe::Failed { stderr }) => format!("unhealthy: {}", first_line(&stderr)), + Ok(Probe::Unknown { reason }) => format!("unknown: {}", first_line(&reason)), + Ok(Probe::MissingBinary { binary, .. }) => { + format!("unhealthy: `{binary}` does not exist in the container") + } + Err(error) => format!("cannot probe: {}", first_line(&error.to_string())), + } +} + +fn first_line(text: &str) -> &str { + text.trim().lines().next().unwrap_or("") +} diff --git a/crates/top/src/ui.rs b/crates/top/src/ui.rs new file mode 100644 index 0000000..d780257 --- /dev/null +++ b/crates/top/src/ui.rs @@ -0,0 +1,277 @@ +//! Drawing. Everything shown is in [`App`]; nothing here calls the runtime. + +use ratatui::layout::{Constraint, Layout, Rect}; +use ratatui::style::{Color, Modifier, Style, Stylize}; +use ratatui::text::{Line, Span}; +use ratatui::widgets::{Block, Cell, Paragraph, Row as TableRow, Table}; +use ratatui::Frame; + +use crate::app::App; +use crate::model::{Row, NOT_CREATED, NO_CHECK, NO_HEALTH}; + +/// Rows of the events pane, inside its border. +const EVENT_LINES: u16 = 5; + +pub fn draw(frame: &mut Frame, app: &mut App) { + // Header, services (sized to fit), events, logs (the rest), footer. + let table_height = u16::try_from(app.rows.len()).unwrap_or(u16::MAX).max(1) + 3; + let [header, services, events, logs, footer] = frame.area().layout(&Layout::vertical([ + Constraint::Length(1), + Constraint::Length(table_height), + Constraint::Length(EVENT_LINES + 2), + Constraint::Fill(1), + Constraint::Length(1), + ])); + + draw_header(frame, header, app); + draw_services(frame, services, app); + draw_events(frame, events, app); + draw_logs(frame, logs, app); + draw_footer(frame, footer, app); +} + +fn draw_header(frame: &mut Frame, area: Rect, app: &App) { + let mut spans = vec![ + Span::from(format!(" {} ", app.declared.project)) + .bold() + .reversed(), + Span::from(" "), + ]; + match app.supervisor_pid { + Some(pid) => spans.push(Span::from(format!("supervisor pid {pid}")).green()), + None => spans.push(Span::from("no supervisor: health unknown").yellow()), + } + if let Some(updated) = &app.updated_at { + spans.push(Span::from(format!(" · updated {updated}")).dark_gray()); + } + if let Some(error) = &app.runtime_error { + spans.push(Span::from(" · runtime: ").red()); + spans.push(Span::from(error.clone()).red()); + } + frame.render_widget(Line::from(spans), area); +} + +fn draw_services(frame: &mut Frame, area: Rect, app: &mut App) { + let header = TableRow::new([ + "SERVICE", "STATE", "HEALTH", "UPTIME", "RESTARTS", "POLICY", "PORTS", "IMAGE", + ]) + .style(Style::new().bold()); + + let rows: Vec = app.rows.iter().map(service_row).collect(); + let table = Table::new( + rows, + [ + Constraint::Length(14), + Constraint::Length(11), + Constraint::Length(10), + Constraint::Length(8), + Constraint::Length(10), + Constraint::Length(8), + Constraint::Length(24), + Constraint::Fill(1), + ], + ) + .header(header) + .column_spacing(1) + .row_highlight_style(Style::new().add_modifier(Modifier::REVERSED)) + .highlight_symbol("▶ ") + .block(Block::bordered().title(" services ")); + + if app.rows.is_empty() { + let message = if app.updated_at.is_some() { + "no containers and no declared services for this project" + } else { + "waiting for the first poll…" + }; + frame.render_widget( + Paragraph::new(message) + .dark_gray() + .block(Block::bordered().title(" services ")), + area, + ); + return; + } + frame.render_stateful_widget(table, area, &mut app.table); +} + +fn service_row(row: &Row) -> TableRow<'static> { + let state_style = match row.state.as_str() { + "running" => Style::new().green(), + NOT_CREATED => Style::new().dark_gray(), + _ => Style::new().red(), + }; + let health_style = match row.health.as_str() { + "healthy" => Style::new().green(), + "unhealthy" => Style::new().red(), + "starting" => Style::new().yellow(), + NO_HEALTH | NO_CHECK => Style::new().dark_gray(), + _ => Style::new().yellow(), + }; + let restarts = match (row.restarts, row.gave_up) { + (count, true) => Cell::from(format!("{count} gave up")).red(), + (0, false) => Cell::from("0").dark_gray(), + (count, false) => Cell::from(count.to_string()).yellow(), + }; + TableRow::new([ + Cell::from(row.service.clone()).bold(), + Cell::from(row.state.clone()).style(state_style), + Cell::from(row.health.clone()).style(health_style), + Cell::from(row.uptime.clone().unwrap_or_else(|| "—".to_owned())), + restarts, + Cell::from(row.policy.clone().unwrap_or_else(|| "—".to_owned())), + Cell::from(row.ports.join(", ")), + Cell::from(row.image.clone().unwrap_or_default()).dark_gray(), + ]) +} + +fn draw_events(frame: &mut Frame, area: Rect, app: &App) { + let lines: Vec = if app.events.is_empty() { + vec![Line::from("no changes since top started").dark_gray()] + } else { + app.events + .iter() + .take(usize::from(EVENT_LINES)) + .map(|event| { + Line::from(vec![ + Span::from(format!("{} ", event.at)).dark_gray(), + Span::from(format!("{} ", event.service)).bold(), + Span::from(event.text.clone()), + ]) + }) + .collect() + }; + frame.render_widget( + Paragraph::new(lines).block(Block::bordered().title(" events ")), + area, + ); +} + +fn draw_logs(frame: &mut Frame, area: Rect, app: &App) { + let service = app.selected_service().unwrap_or(""); + let scrolled = if app.log_scroll > 0 { + format!(" · {} lines up", app.log_scroll) + } else { + String::new() + }; + let block = Block::bordered().title(format!(" logs · {service}{scrolled} ")); + + let Some(logs) = &app.logs else { + let row = app.rows.iter().find(|row| row.service == service); + let message = match row { + Some(row) if row.container.is_none() => "the container has not been created", + Some(_) => "fetching…", + None => "", + }; + frame.render_widget(Paragraph::new(message).dark_gray().block(block), area); + return; + }; + if let Some(error) = &logs.error { + frame.render_widget(Paragraph::new(error.clone()).red().block(block), area); + return; + } + + // The tail that fits, moved up by the scroll. Lines are clipped rather than + // wrapped, so one line of log is one line of screen and the arithmetic holds. + let height = usize::from(area.height.saturating_sub(2)); + let end = logs.lines.len().saturating_sub(app.log_scroll); + let start = end.saturating_sub(height); + let lines: Vec = logs.lines[start..end] + .iter() + .map(|line| Line::from(line.clone())) + .collect(); + frame.render_widget(Paragraph::new(lines).block(block), area); +} + +fn draw_footer(frame: &mut Frame, area: Rect, app: &App) { + let keys = Span::from(" q quit · ↑↓ select · p probe · r refresh · PgUp/PgDn/End logs ") + .style(Style::new().fg(Color::Black).bg(Color::Gray)); + let mut spans = vec![keys]; + if let Some(note) = &app.note { + spans.push(Span::from(format!(" {note}"))); + } + frame.render_widget(Line::from(spans), area); +} + +#[cfg(test)] +mod tests { + use super::draw; + use crate::app::App; + use crate::model::Declared; + use crate::poll::{Logs, Snapshot, Update}; + use container_compose_runtime::ObservedContainer; + use ratatui::backend::TestBackend; + use ratatui::Terminal; + + fn captured() -> Vec { + let text = std::fs::read_to_string(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../fixtures/real/container-list.json" + )) + .unwrap(); + serde_json::from_str(&text).unwrap() + } + + /// The screen as text, one line per terminal row, for assertions and for + /// reading what was drawn. + pub(crate) fn screen(app: &mut App, width: u16, height: u16) -> String { + let mut terminal = Terminal::new(TestBackend::new(width, height)).unwrap(); + terminal.draw(|frame| draw(frame, app)).unwrap(); + let buffer = terminal.backend().buffer(); + (0..height) + .map(|y| { + (0..width) + .map(|x| buffer[(x, y)].symbol()) + .collect::() + .trim_end() + .to_owned() + }) + .collect::>() + .join("\n") + } + + fn app_with_logs() -> App { + let project = captured()[0].project().unwrap().to_owned(); + let mut app = App::new(Declared::bare(project)); + app.apply(Update::Snapshot(Snapshot { + at_secs: 1_792_830_352, + containers: Ok(captured()), + cache: None, + logs: None, + })); + app.apply(Update::Snapshot(Snapshot { + at_secs: 1_792_830_353, + containers: Ok(captured()), + cache: None, + logs: Some(Logs { + service: "api".to_owned(), + lines: vec!["capture-hello".to_owned()], + error: None, + }), + })); + app + } + + #[test] + fn the_screen_shows_services_events_and_the_selected_services_logs() { + let mut app = app_with_logs(); + let text = screen(&mut app, 120, 24); + println!("{text}"); + + assert!(text.contains("no supervisor: health unknown")); + assert!(text.contains("SERVICE")); + assert!(text.contains("▶ api")); + assert!(text.contains("worker")); + assert!(text.contains("running")); + assert!(text.contains("stopped")); + assert!(text.contains("logs · api")); + assert!(text.contains("capture-hello")); + assert!(text.contains("q quit")); + } + + #[test] + fn a_narrow_terminal_still_draws() { + let mut app = app_with_logs(); + let text = screen(&mut app, 40, 12); + assert!(text.contains("api")); + } +} diff --git a/docs/01-model.md b/docs/01-model.md index c96def1..333bdba 100644 --- a/docs/01-model.md +++ b/docs/01-model.md @@ -215,7 +215,10 @@ healthcheck can never pass, and the message says which binary is missing. Health is a property of the supervision session, not of the container, because nothing in the runtime records it. The supervisor writes a small per-project cache — `~/Library/Caches/container-compose/.json`, holding its pid and -the last result per service — and `ps` reads it. A missing file, a pid that is +the last result per service, with the restarts it has issued for each and whether +it gave up — and `ps` and `top` read it. The restart count is history for a +reader and never resets; the tick's attempt counter, which does reset after 60 s +of healthy uptime, stays in memory. A missing file, a pid that is gone, or a config-hash mismatch means `unknown`, never a stale "healthy". The cache is never authority: `up` readiness uses in-process state, deleting the file changes only what `ps` prints, and failing to write it warns rather than diff --git a/docs/03-cli.md b/docs/03-cli.md index bd6c4d1..1f24305 100644 --- a/docs/03-cli.md +++ b/docs/03-cli.md @@ -210,6 +210,50 @@ it. `checked_at` and `source` (`supervisor`, `probe`, or `none`), not a formatted string. +## `top` + +```sh +container-compose top [--interval 1s] [-n 200] +``` + +`ps`, kept live in the terminal. One row per declared service in start order — +including any not yet created — then any undeclared container of the project: + +``` + midnight-lab supervisor pid 5630 · updated 01:50:28Z + SERVICE STATE HEALTH UPTIME RESTARTS POLICY PORTS +▶ node running healthy 32s 0 no 127.0.0.1:9944->9944 + indexer running no check 12s 1 on-stop 127.0.0.1:8088->8088 +``` + +Below the table, an events pane lists what changed between polls — a state, a +health, a restart, a give-up — newest first, and a logs pane follows the +selected service's last `-n` lines. + +| key | does | +|---|---| +| `↑` `↓` (`k` `j`) | select a service; its logs follow | +| `p` | run the selected service's healthcheck now, as `ps --probe` does | +| `r` | poll now | +| `PgUp` `PgDn` `End` | scroll the logs; `End` returns to the tail | +| `q`, `Esc`, `Ctrl-C` | quit | + +`top` reads what `ps` reads, and nothing else: the runtime's container list and a +live supervisor's health cache. It takes no lock and starts or stops nothing, so +it runs beside `up`. `HEALTH` follows `ps`'s rules, except that a running service +declaring no healthcheck shows `no check` rather than `unknown`. `RESTARTS` comes +from the cache, so without a live supervisor it is `0`. + +It polls with `container list` alone, not the full observation `plan` and `ps` +use, so it does not gate on `container system status`. That status answers +`unregistered` to any launchd session other than the one that started the runtime +— an SSH login, typically — while `list` works from anywhere, so `top` works over +SSH where `ps` refuses. + +Like `ps`, it works without the stack file, from the project's containers alone; +then there is no on-demand probe. Without a terminal it refuses (exit 1) and +points at `ps --json`. + ## `logs` ```sh From 786c81c1385d7f199aab1db57503433095114c6a Mon Sep 17 00:00:00 2001 From: Damon Oehlman Date: Mon, 26 Oct 2026 12:39:06 +1000 Subject: [PATCH 3/4] supervisor sampling --- README.md | 3 +- crates/cli/src/cli.rs | 210 +++++++++++++ crates/cli/tests/plan.rs | 1 + crates/cli/tests/smoke.rs | 1 + crates/cli/tests/up.rs | 138 ++++++++- crates/runtime/src/lib.rs | 2 +- crates/runtime/src/observe.rs | 41 +++ crates/runtime/tests/captured.rs | 15 +- crates/supervisor/src/history.rs | 442 ++++++++++++++++++++++++++++ crates/supervisor/src/lib.rs | 4 + crates/supervisor/src/run.rs | 41 ++- crates/supervisor/src/sampler.rs | 198 +++++++++++++ crates/top/src/app.rs | 11 + crates/top/src/lib.rs | 1 + crates/top/src/model.rs | 213 ++++++++++++-- crates/top/src/poll.rs | 27 +- crates/top/src/ui.rs | 84 +++++- dev/fake-container/src/main.rs | 35 ++- docs/01-model.md | 11 +- docs/03-cli.md | 65 +++- fixtures/real/capture.txt | 2 + fixtures/real/container-stats.json | 1 + scripts/capture-runtime-fixtures.sh | 4 + 23 files changed, 1503 insertions(+), 47 deletions(-) create mode 100644 crates/supervisor/src/history.rs create mode 100644 crates/supervisor/src/sampler.rs create mode 100644 fixtures/real/container-stats.json diff --git a/README.md b/README.md index fb9d0a6..dd741a1 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,8 @@ domain was registered do not get a name — `down` then `up` after setting it. | `up` | create and start in dependency order, then supervise until Ctrl-C | | `down` | stop and delete what this project owns; `--volumes`, `--timeout` | | `ps` | what is running, with health; `--probe`, `--json` | -| `top` | `ps` kept live in the terminal, with restarts, changes and the selected service's logs | +| `top` | `ps` kept live in the terminal, with CPU and memory, restarts, changes and the selected service's logs | +| `stats` | CPU, memory, network and disk per service over a window, from the supervisor's history; `--since`, `--json` | | `logs`, `exec`, `run` | the runtime's own, scoped to the stack | | `build`, `pull` | the images the stack declares | diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index a362871..03cbb98 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -25,6 +25,7 @@ use container_compose_stack::resolve::{ use container_compose_stack::{self as stack, Options}; use container_compose_supervisor::cache; use container_compose_supervisor::clock; +use container_compose_supervisor::history; use container_compose_supervisor::lock::{self, LockError}; use container_compose_supervisor::{ run as supervise, Options as SupervisorOptions, SupervisorError, @@ -192,6 +193,15 @@ pub enum Command { #[arg(short = 'n', long, default_value = "200")] lines: u32, }, + /// Resource usage per service over a window, from the supervisor's history. + Stats { + /// How far back to summarise; the history keeps seven days. + #[arg(long, default_value = "1h", value_parser = parse_span)] + since: Span, + /// Emit the machine-readable document. + #[arg(long)] + json: bool, + }, /// Print a service's logs. Logs { /// Services to show; default is all. @@ -238,6 +248,7 @@ pub fn run(cli: Cli) -> Result<(), Error> { } => down_command(&cli, *volumes, *timeout, *dry_run), Command::Ps { json, probe } => ps_command(&cli, *json, *probe), Command::Top { interval, lines } => top_command(&cli, *interval, *lines), + Command::Stats { since, json } => stats_command(&cli, *since, *json), Command::Logs { services, follow, @@ -396,6 +407,7 @@ fn up_command(cli: &Cli, flags: &UpFlags<'_>) -> Result<(), Error> { stop_timeout: timeout_secs(flags.timeout), no_stop_on_exit: flags.no_stop_on_exit, verbose: cli.verbose, + sample_interval: Some(container_compose_supervisor::history::SAMPLE_INTERVAL), }; supervise( &stack, @@ -575,6 +587,204 @@ fn top_command(cli: &Cli, interval: Span, lines: u32) -> Result<(), Error> { .map_err(|error| Error::Runtime(error.to_string())) } +#[derive(Debug, Serialize)] +struct StatsDocument { + project: String, + since_secs: u64, + samples: usize, + first_at: Option, + last_at: Option, + /// Intervals between samples longer than two sampling periods: times no + /// supervisor was running, or the runtime gave no readings. + gaps: usize, + services: Vec, +} + +fn stats_command(cli: &Cli, since: Span, json: bool) -> Result<(), Error> { + // Like `ps`, the stack file only orders the rows; history outlives it. + let stack = if cli.file.exists() { + Some(stack::load( + &cli.file, + &Options { + project_name: cli.project_name.clone(), + }, + )?) + } else { + None + }; + let project = stack + .as_ref() + .map(|stack| stack.name.clone()) + .unwrap_or_else(|| project_from_path(cli)); + let path = history::path(&project).ok_or_else(|| { + Error::Internal(format!( + "neither {} nor HOME is set", + history::HISTORY_DIR_ENV + )) + })?; + + let since_secs = u64::try_from(since.millis() / 1_000).unwrap_or(u64::MAX); + let cutoff = (clock::wall_ms() / 1_000).saturating_sub(since_secs); + let samples: Vec = history::read_at(&path) + .into_iter() + .filter(|sample| sample.at >= cutoff) + .collect(); + + // Declared services in start order, then anything else the history holds. + let mut services: Vec = stack + .as_ref() + .map(|stack| stack.order.clone()) + .unwrap_or_default(); + let mut recorded: Vec = samples + .iter() + .flat_map(|sample| sample.services.keys().cloned()) + .collect::>() + .into_iter() + .filter(|service| !services.contains(service)) + .collect(); + services.append(&mut recorded); + + let gap_secs = 2 * history::SAMPLE_INTERVAL.as_secs(); + let document = StatsDocument { + project: project.clone(), + since_secs, + samples: samples.len(), + first_at: samples + .first() + .map(|sample| clock::format_rfc3339_seconds(sample.at)), + last_at: samples + .last() + .map(|sample| clock::format_rfc3339_seconds(sample.at)), + gaps: samples + .windows(2) + .filter(|pair| pair[1].at.saturating_sub(pair[0].at) > gap_secs) + .count(), + services: services + .iter() + .filter_map(|service| history::summarize(&samples, service)) + .collect(), + }; + + if json { + let rendered = serde_json::to_string_pretty(&document) + .map_err(|error| Error::Internal(format!("cannot render stats: {error}")))?; + println!("{rendered}"); + return Ok(()); + } + print!("{}", render_stats(&document, &services, since)); + Ok(()) +} + +fn render_stats(document: &StatsDocument, services: &[String], since: Span) -> String { + let window = render_span(since); + let (Some(first), Some(last)) = (&document.first_at, &document.last_at) else { + return format!( + "no history for {} in the last {window}. A supervisor records a sample every {}s \ + while `up` runs.\n", + document.project, + history::SAMPLE_INTERVAL.as_secs() + ); + }; + + let gaps = match document.gaps { + 0 => String::new(), + 1 => ", 1 gap".to_owned(), + count => format!(", {count} gaps"), + }; + let mut out = format!( + "{} · last {window} · {} samples from {first} to {last}{gaps}\n\n", + document.project, document.samples + ); + + let pct = |value: Option| value.map_or_else(|| "—".to_owned(), history::format_pct); + let mut rows: Vec> = vec![[ + "SERVICE", + "READINGS", + "CPU AVG", + "CPU PEAK", + "MEM AVG", + "MEM PEAK", + "LIMIT", + "NET IN", + "NET OUT", + "DISK READ", + "DISK WRITE", + "RESTARTS", + ] + .map(str::to_owned) + .to_vec()]; + for service in services { + let row = match document + .services + .iter() + .find(|summary| &summary.service == service) + { + Some(summary) => vec![ + service.clone(), + summary.readings.to_string(), + pct(summary.cpu_avg_pct), + pct(summary.cpu_peak_pct), + history::format_bytes(summary.mem_avg), + history::format_bytes(summary.mem_peak), + history::format_bytes(summary.mem_limit), + history::format_bytes(summary.net_rx), + history::format_bytes(summary.net_tx), + history::format_bytes(summary.blk_read), + history::format_bytes(summary.blk_write), + summary.restarts.to_string(), + ], + None => { + let mut row = vec![service.clone(), "0".to_owned()]; + row.extend(std::iter::repeat_n("—".to_owned(), 10)); + row + } + }; + rows.push(row); + } + out.push_str(&render_columns(&rows)); + out +} + +/// Left-aligned columns two spaces apart; the last column is not padded. +fn render_columns(rows: &[Vec]) -> String { + let columns = rows.iter().map(Vec::len).max().unwrap_or(0); + let widths: Vec = (0..columns) + .map(|index| { + rows.iter() + .filter_map(|row| row.get(index)) + .map(|cell| cell.chars().count()) + .max() + .unwrap_or(0) + }) + .collect(); + let mut out = String::new(); + for row in rows { + for (index, cell) in row.iter().enumerate() { + out.push_str(cell); + if index + 1 < row.len() { + let padding = widths[index].saturating_sub(cell.chars().count()) + 2; + out.push_str(&" ".repeat(padding)); + } + } + out.push('\n'); + } + out +} + +/// `1h`, `90m`, `7d`, `45s`: the largest whole unit. +fn render_span(span: Span) -> String { + let seconds = u64::try_from(span.millis() / 1_000).unwrap_or(u64::MAX); + if seconds % 86_400 == 0 && seconds > 0 { + format!("{}d", seconds / 86_400) + } else if seconds % 3_600 == 0 && seconds > 0 { + format!("{}h", seconds / 3_600) + } else if seconds % 60 == 0 && seconds > 0 { + format!("{}m", seconds / 60) + } else { + format!("{seconds}s") + } +} + struct TerminalFlags { interactive: bool, tty: bool, diff --git a/crates/cli/tests/plan.rs b/crates/cli/tests/plan.rs index b4150fa..d7f9b37 100644 --- a/crates/cli/tests/plan.rs +++ b/crates/cli/tests/plan.rs @@ -72,6 +72,7 @@ impl Fake { ) .env("FAKE_CONTAINER_DIR", &self.world) .env("CONTAINER_COMPOSE_CACHE_DIR", &self.cache) + .env("CONTAINER_COMPOSE_HISTORY_DIR", self.cache.join("history")) .args(args); command } diff --git a/crates/cli/tests/smoke.rs b/crates/cli/tests/smoke.rs index aead922..7de6054 100644 --- a/crates/cli/tests/smoke.rs +++ b/crates/cli/tests/smoke.rs @@ -73,6 +73,7 @@ impl Smoke { fn command(&self, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_container-compose")) .env("CONTAINER_COMPOSE_CACHE_DIR", &self.cache) + .env("CONTAINER_COMPOSE_HISTORY_DIR", self.cache.join("history")) .args([ "-f", self.stack.to_str().expect("utf-8"), diff --git a/crates/cli/tests/up.rs b/crates/cli/tests/up.rs index 274c31a..4f4cd6d 100644 --- a/crates/cli/tests/up.rs +++ b/crates/cli/tests/up.rs @@ -1,4 +1,4 @@ -//! `up`, `down` and `ps` through the real binary, against the fake runtime. +//! `up`, `down`, `ps` and `stats` through the real binary, against the fake runtime. //! //! The fake is the `container` on the child's `PATH`, so these tests exercise //! the same argv, the same polling and the same ownership filters a real runtime @@ -18,6 +18,7 @@ use container_compose_runtime::{ labels, ContainerConfiguration, ContainerImage, ContainerStatus, Observed, ObservedContainer, }; use container_compose_supervisor::cache::{write_at, CachedHealth, HealthCache}; +use container_compose_supervisor::history::{append_at, Reading, Sample}; #[derive(Debug, Deserialize)] struct Call { @@ -101,6 +102,8 @@ impl Fake { ) .env("FAKE_CONTAINER_DIR", &self.world) .env("CONTAINER_COMPOSE_CACHE_DIR", &self.cache) + // A supervised test must never append to the user's real history. + .env("CONTAINER_COMPOSE_HISTORY_DIR", self.cache.join("history")) .args(args); command } @@ -1174,6 +1177,7 @@ fn spawn_up(fake: &Fake, path: &str, extra: &[&str]) -> std::process::Child { ) .env("FAKE_CONTAINER_DIR", &fake.world) .env("CONTAINER_COMPOSE_CACHE_DIR", &fake.cache) + .env("CONTAINER_COMPOSE_HISTORY_DIR", fake.cache.join("history")) .args(["-f", path, "up"]) .args(extra) .stdout(std::process::Stdio::piped()) @@ -1658,3 +1662,135 @@ volumes = ["data:/data"] assert_eq!(run.status.code(), Some(3), "stderr: {}", stderr(&run)); assert!(stderr(&run).contains("named volume"), "{}", stderr(&run)); } + +#[derive(Debug, Deserialize)] +struct StatsDoc { + project: String, + samples: usize, + gaps: usize, + services: Vec, +} + +#[derive(Debug, Deserialize)] +struct StatsSummaryDoc { + service: String, + readings: usize, + cpu_avg_pct: Option, + mem_peak: u64, + mem_limit: u64, +} + +/// Three samples of the `echo` service 30 s apart, then one after a gap, all +/// ending a minute ago. The counters are the captured midnight node reading +/// (`fixtures/real/container-stats.json`), advanced by 3 s of CPU per sample. +fn write_history(fake: &Fake, now: u64) { + let captured: Vec = + serde_json::from_str(&fixture_file("real/container-stats.json")).expect("captured stats"); + let node = captured + .iter() + .find(|reading| reading.id == "midnight-lab-node") + .expect("the node's reading"); + let path = fake.cache.join("history").join("minimal.jsonl"); + for (index, at) in [now - 300, now - 270, now - 240, now - 60] + .into_iter() + .enumerate() + { + let reading = Reading { + started: Some("2026-10-26T01:49:56Z".to_owned()), + cpu_usec: node.cpu_usage_usec + 3_000_000 * index as u64, + mem: node.memory_usage_bytes, + mem_limit: node.memory_limit_bytes, + net_rx: node.network_rx_bytes, + net_tx: node.network_tx_bytes, + blk_read: node.block_read_bytes, + blk_write: node.block_write_bytes, + pids: node.num_processes, + }; + append_at( + &path, + &Sample { + at, + services: BTreeMap::from([("echo".to_owned(), reading)]), + }, + ) + .expect("append a sample"); + } +} + +fn fixture_file(relative: &str) -> String { + fs::read_to_string( + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../fixtures") + .join(relative), + ) + .expect("a fixture") +} + +fn epoch_now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("after 1970") + .as_secs() +} + +#[test] +fn stats_summarises_the_history_window() { + let fake = Fake::new(); + write_history(&fake, epoch_now()); + + let listed = fake + .command(&[ + "-f", + &fixture("minimal"), + "stats", + "--since", + "10m", + "--json", + ]) + .output() + .expect("runs"); + assert!(listed.status.success(), "stderr: {}", stderr(&listed)); + let document: StatsDoc = serde_json::from_slice(&listed.stdout).expect("a stats document"); + assert_eq!(document.project, "minimal"); + assert_eq!(document.samples, 4); + assert_eq!( + document.gaps, 1, + "180 s between the third and fourth samples" + ); + + let echo = &document.services[0]; + assert_eq!(echo.service, "echo"); + assert_eq!(echo.readings, 4); + // 3 s of CPU per 30 s is 10% of a core; across the gap it is 3 s in 180 s. + let expected = (10.0 + 10.0 + 3.0 / 180.0 * 100.0) / 3.0; + let average = echo.cpu_avg_pct.expect("rates exist"); + assert!((average - expected).abs() < 1e-9, "{average} vs {expected}"); + assert_eq!(echo.mem_peak, 332_886_016); + assert_eq!(echo.mem_limit, 1 << 30); + + let table = fake + .command(&["-f", &fixture("minimal"), "stats", "--since", "10m"]) + .output() + .expect("runs"); + let text = String::from_utf8_lossy(&table.stdout); + assert!(text.contains("minimal · last 10m · 4 samples"), "{text}"); + assert!(text.contains("1 gap"), "{text}"); + assert!(text.contains("317 MiB"), "{text}"); +} + +#[test] +fn stats_outside_the_window_says_there_is_no_history() { + let fake = Fake::new(); + write_history(&fake, epoch_now() - 7_200); + + let table = fake + .command(&["-f", &fixture("minimal"), "stats", "--since", "1h"]) + .output() + .expect("runs"); + assert!(table.status.success(), "stderr: {}", stderr(&table)); + let text = String::from_utf8_lossy(&table.stdout); + assert!( + text.starts_with("no history for minimal in the last 1h"), + "{text}" + ); +} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 2e4bcc7..16a5b3a 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -15,6 +15,6 @@ pub use command::{Command, Completion, Output, RuntimeError}; pub use observe::{ normalize_image_reference, observe, ContainerConfiguration, ContainerImage, ContainerStatus, ImageConfiguration, NetworkConfiguration, NetworkStatus, Observed, ObservedContainer, - ObservedImage, ObservedNetwork, ObservedVolume, VolumeConfiguration, + ObservedImage, ObservedNetwork, ObservedStats, ObservedVolume, VolumeConfiguration, }; pub use probe::{probe, Probe}; diff --git a/crates/runtime/src/observe.rs b/crates/runtime/src/observe.rs index c36d6d9..0e10614 100644 --- a/crates/runtime/src/observe.rs +++ b/crates/runtime/src/observe.rs @@ -156,6 +156,47 @@ pub fn observe(program: &str) -> Result { }) } +/// One container's resource counters from `container stats --format json`. +/// +/// CPU, network and block I/O are cumulative since the container started, so a +/// rate is a difference between two readings; memory and processes are +/// instantaneous. The shape is `fixtures/real/container-stats.json`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ObservedStats { + pub id: String, + #[serde(default)] + pub cpu_usage_usec: u64, + #[serde(default)] + pub memory_usage_bytes: u64, + #[serde(default)] + pub memory_limit_bytes: u64, + #[serde(default)] + pub network_rx_bytes: u64, + #[serde(default)] + pub network_tx_bytes: u64, + #[serde(default)] + pub block_read_bytes: u64, + #[serde(default)] + pub block_write_bytes: u64, + #[serde(default)] + pub num_processes: u64, +} + +/// `stats --no-stream`, for every running container, parsed. +/// +/// No container ids are passed: naming one that is not running fails the whole +/// call, and the caller filters by name anyway. The runtime takes two samples +/// two seconds apart before printing, so this blocks for at least that long. +/// A container whose statistics call fails inside the runtime is left out +/// rather than reported, so absence means "no reading", not zero. +pub fn stats(program: &str) -> Result, RuntimeError> { + parse( + "`container stats`", + &run(program, &["stats", "--no-stream", "--format", "json"])?, + ) +} + /// `list --all`, parsed. The start phase polls this rather than calling the /// full `observe`, which is four more subprocesses per poll. pub fn containers(program: &str) -> Result, RuntimeError> { diff --git a/crates/runtime/tests/captured.rs b/crates/runtime/tests/captured.rs index cad76d2..91e5a85 100644 --- a/crates/runtime/tests/captured.rs +++ b/crates/runtime/tests/captured.rs @@ -13,7 +13,7 @@ use std::fs; use std::path::PathBuf; use container_compose_runtime::{ - ObservedContainer, ObservedImage, ObservedNetwork, ObservedVolume, + ObservedContainer, ObservedImage, ObservedNetwork, ObservedStats, ObservedVolume, }; use serde::Deserialize; @@ -161,3 +161,16 @@ fn the_captured_images_parse() { .expect("the image the capture used"); assert!(!alpine.id.is_empty()); } + +#[test] +fn the_captured_stats_parse() { + let stats: Vec = parse("container-stats.json"); + let node = stats + .iter() + .find(|reading| reading.id == "midnight-lab-node") + .expect("the node's reading"); + assert!(node.cpu_usage_usec > 0, "cumulative CPU time is reported"); + assert!(node.memory_usage_bytes > 0); + assert_eq!(node.memory_limit_bytes, 1024 * 1024 * 1024); + assert!(node.num_processes > 0); +} diff --git a/crates/supervisor/src/history.rs b/crates/supervisor/src/history.rs new file mode 100644 index 0000000..5f1e9d4 --- /dev/null +++ b/crates/supervisor/src/history.rs @@ -0,0 +1,442 @@ +//! Resource history: a low-resolution record of each service's usage. +//! +//! While a supervisor runs it appends one [`Sample`] every 30 seconds to +//! `~/Library/Application Support/container-compose/history/.jsonl`, +//! one JSON line per sample, and drops lines older than seven days. The +//! readings are the runtime's raw counters; rates are derived when read, so +//! the file stays a plain record of what the runtime said. +//! +//! A gap in the file means no supervisor was running. A service missing from +//! a sample means the runtime gave no reading for it — never zero usage. +//! +//! `CONTAINER_COMPOSE_HISTORY_DIR` overrides the directory; the tests use the +//! `*_at` functions with a temporary path instead. + +use std::collections::BTreeMap; +use std::fs::{self, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +pub const HISTORY_DIR_ENV: &str = "CONTAINER_COMPOSE_HISTORY_DIR"; +/// How often a supervisor samples. +pub const SAMPLE_INTERVAL: Duration = Duration::from_secs(30); +/// How long samples are kept. +pub const RETENTION_SECS: u64 = 7 * 86_400; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Sample { + /// Epoch seconds. + pub at: u64, + /// Readings by service name. + pub services: BTreeMap, +} + +/// One container's counters, as the runtime reported them. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Reading { + /// The container's start time. A different value from the previous + /// reading means the counters restarted from zero. + pub started: Option, + pub cpu_usec: u64, + pub mem: u64, + pub mem_limit: u64, + pub net_rx: u64, + pub net_tx: u64, + pub blk_read: u64, + pub blk_write: u64, + pub pids: u64, +} + +pub fn dir() -> Option { + if let Some(overridden) = std::env::var_os(HISTORY_DIR_ENV) { + return Some(PathBuf::from(overridden)); + } + std::env::var_os("HOME").map(|home| { + PathBuf::from(home).join("Library/Application Support/container-compose/history") + }) +} + +pub fn path(project: &str) -> Option { + Some(dir()?.join(format!("{project}.jsonl"))) +} + +pub fn append_at(path: &Path, sample: &Sample) -> io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut line = serde_json::to_vec(sample) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + line.push(b'\n'); + // One `write` of one line: an append-mode write of a few hundred bytes + // lands whole, so a reader never sees half a sample. + OpenOptions::new() + .create(true) + .append(true) + .open(path)? + .write_all(&line) +} + +/// Every sample in the file. A line that does not parse — a truncated last +/// line after a crash — is skipped rather than failing the read. +pub fn read_at(path: &Path) -> Vec { + fs::read_to_string(path) + .map(|text| parse_lines(&text, false)) + .unwrap_or_default() +} + +/// The samples in the last `max_bytes` of the file, for a reader that only +/// wants the recent past and polls often. +pub fn read_tail_at(path: &Path, max_bytes: u64) -> Vec { + let Ok(mut file) = fs::File::open(path) else { + return Vec::new(); + }; + let Ok(length) = file.metadata().map(|metadata| metadata.len()) else { + return Vec::new(); + }; + let start = length.saturating_sub(max_bytes); + if file.seek(SeekFrom::Start(start)).is_err() { + return Vec::new(); + } + let mut text = String::new(); + if file.read_to_string(&mut text).is_err() { + return Vec::new(); + } + // Starting mid-file lands mid-line; the first line is partial. + parse_lines(&text, start > 0) +} + +fn parse_lines(text: &str, skip_first: bool) -> Vec { + text.lines() + .skip(usize::from(skip_first)) + .filter_map(|line| serde_json::from_str(line).ok()) + .collect() +} + +/// Drop samples older than `now - retention`, rewriting the file atomically. +pub fn prune_at(path: &Path, now: u64, retention: u64) -> io::Result<()> { + let samples = read_at(path); + let cutoff = now.saturating_sub(retention); + if samples.iter().all(|sample| sample.at >= cutoff) { + return Ok(()); + } + let mut text = String::new(); + for sample in samples.iter().filter(|sample| sample.at >= cutoff) { + let line = serde_json::to_string(sample) + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + text.push_str(&line); + text.push('\n'); + } + let temporary = path.with_extension("jsonl.tmp"); + fs::write(&temporary, text)?; + fs::rename(&temporary, path) +} + +/// One service at one sample, with rates over the interval since the reading +/// before it. Rates are `None` on the first reading after a start or a gap in +/// the service's readings, where there is nothing to subtract. +#[derive(Debug, Clone, PartialEq)] +pub struct Point { + pub at: u64, + /// CPU over the interval, in percent of one core (so 4 busy cores is 400). + pub cpu_pct: Option, + pub mem: u64, + pub mem_limit: u64, + pub net_rx_per_sec: Option, + pub net_tx_per_sec: Option, + pub blk_read_per_sec: Option, + pub blk_write_per_sec: Option, + pub pids: u64, + /// This reading starts a new run: the container restarted, or it is the + /// first reading of the service in the samples given. + pub starts_run: bool, +} + +/// The service's readings as points, oldest first. +pub fn series(samples: &[Sample], service: &str) -> Vec { + let mut points = Vec::new(); + let mut previous: Option<(u64, &Reading)> = None; + for sample in samples { + let Some(reading) = sample.services.get(service) else { + // No reading: the next one cannot be differenced across the hole. + previous = None; + continue; + }; + let continuous = previous.filter(|(at, before)| { + *at < sample.at + && before.started == reading.started + && reading.cpu_usec >= before.cpu_usec + && reading.net_rx >= before.net_rx + && reading.net_tx >= before.net_tx + && reading.blk_read >= before.blk_read + && reading.blk_write >= before.blk_write + }); + let rate = |now: u64, then: u64| -> Option { + continuous.map(|(at, _)| (now - then) as f64 / (sample.at - at) as f64) + }; + let before = continuous.map(|(_, before)| before); + points.push(Point { + at: sample.at, + cpu_pct: before.and_then(|before| { + rate(reading.cpu_usec, before.cpu_usec).map(|usec_per_sec| usec_per_sec / 10_000.0) + }), + mem: reading.mem, + mem_limit: reading.mem_limit, + net_rx_per_sec: before.and_then(|before| rate(reading.net_rx, before.net_rx)), + net_tx_per_sec: before.and_then(|before| rate(reading.net_tx, before.net_tx)), + blk_read_per_sec: before.and_then(|before| rate(reading.blk_read, before.blk_read)), + blk_write_per_sec: before.and_then(|before| rate(reading.blk_write, before.blk_write)), + pids: reading.pids, + starts_run: continuous.is_none(), + }); + previous = Some((sample.at, reading)); + } + points +} + +/// A service's usage over the samples given. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct Summary { + pub service: String, + pub readings: usize, + pub first_at: u64, + pub last_at: u64, + pub cpu_avg_pct: Option, + pub cpu_peak_pct: Option, + pub mem_avg: u64, + pub mem_peak: u64, + pub mem_limit: u64, + /// Bytes moved within the window, summed over continuous intervals. + pub net_rx: u64, + pub net_tx: u64, + pub blk_read: u64, + pub blk_write: u64, + /// Restarts visible in the samples: a reading whose start time changed. + pub restarts: usize, +} + +pub fn summarize(samples: &[Sample], service: &str) -> Option { + let points = series(samples, service); + let first = points.first()?; + let last = points.last()?; + + let cpu: Vec = points.iter().filter_map(|point| point.cpu_pct).collect(); + let mem_total: u128 = points.iter().map(|point| u128::from(point.mem)).sum(); + let moved = |value: fn(&Point) -> Option| -> u64 { + points + .windows(2) + .filter_map(|pair| value(&pair[1]).map(|rate| rate * (pair[1].at - pair[0].at) as f64)) + .sum::() + .round() as u64 + }; + + let readings: Vec<&Reading> = samples + .iter() + .filter_map(|sample| sample.services.get(service)) + .collect(); + let restarts = readings + .windows(2) + .filter(|pair| pair[0].started != pair[1].started) + .count(); + + Some(Summary { + service: service.to_owned(), + readings: points.len(), + first_at: first.at, + last_at: last.at, + cpu_avg_pct: (!cpu.is_empty()).then(|| cpu.iter().sum::() / cpu.len() as f64), + cpu_peak_pct: cpu.iter().copied().reduce(f64::max), + mem_avg: u64::try_from(mem_total / points.len() as u128).unwrap_or(u64::MAX), + mem_peak: points.iter().map(|point| point.mem).max().unwrap_or(0), + mem_limit: last.mem_limit, + net_rx: moved(|point| point.net_rx_per_sec), + net_tx: moved(|point| point.net_tx_per_sec), + blk_read: moved(|point| point.blk_read_per_sec), + blk_write: moved(|point| point.blk_write_per_sec), + restarts, + }) +} + +/// `309 MiB`, `1.0 GiB`, `512 B`: binary units, one decimal from GiB up. +pub fn format_bytes(bytes: u64) -> String { + const KIB: f64 = 1024.0; + let value = bytes as f64; + if value >= KIB * KIB * KIB { + format!("{:.1} GiB", value / (KIB * KIB * KIB)) + } else if value >= KIB * KIB { + format!("{:.0} MiB", value / (KIB * KIB)) + } else if value >= KIB { + format!("{:.0} KiB", value / KIB) + } else { + format!("{bytes} B") + } +} + +/// `0.7%`, `12%`, `250%`: one decimal below 10. +pub fn format_pct(pct: f64) -> String { + if pct < 10.0 { + format!("{pct:.1}%") + } else { + format!("{pct:.0}%") + } +} + +#[cfg(test)] +mod tests { + use super::{ + append_at, format_bytes, format_pct, prune_at, read_at, read_tail_at, series, summarize, + Reading, Sample, + }; + use std::collections::BTreeMap; + + fn reading(started: &str, cpu_usec: u64, mem: u64, net_rx: u64) -> Reading { + Reading { + started: Some(started.to_owned()), + cpu_usec, + mem, + mem_limit: 1 << 30, + net_rx, + net_tx: 0, + blk_read: 0, + blk_write: 0, + pids: 4, + } + } + + fn sample(at: u64, node: Option) -> Sample { + Sample { + at, + services: node + .into_iter() + .map(|reading| ("node".to_owned(), reading)) + .collect::>(), + } + } + + const START: &str = "2026-10-26T02:00:00Z"; + const RESTART: &str = "2026-10-26T02:01:10Z"; + + #[test] + fn cpu_is_the_counter_difference_over_the_interval() { + // 15 s of CPU in 30 s is half a core. + let samples = [ + sample(1_000, Some(reading(START, 10_000_000, 100, 0))), + sample(1_030, Some(reading(START, 25_000_000, 200, 3_000))), + ]; + let points = series(&samples, "node"); + assert_eq!(points[0].cpu_pct, None, "nothing to subtract from"); + assert!(points[0].starts_run); + assert_eq!(points[1].cpu_pct, Some(50.0)); + assert_eq!(points[1].net_rx_per_sec, Some(100.0)); + assert_eq!(points[1].mem, 200); + assert!(!points[1].starts_run); + } + + #[test] + fn a_restart_or_a_missing_reading_breaks_the_series() { + let samples = [ + sample(1_000, Some(reading(START, 10_000_000, 100, 0))), + sample(1_030, Some(reading(START, 11_000_000, 100, 0))), + // Restarted: the counters are small again, and the start moved. + sample(1_060, Some(reading(RESTART, 500_000, 50, 0))), + sample(1_090, Some(reading(RESTART, 800_000, 50, 0))), + sample(1_120, None), + sample(1_150, Some(reading(RESTART, 1_100_000, 50, 0))), + ]; + let points = series(&samples, "node"); + let rates: Vec> = points.iter().map(|point| point.cpu_pct).collect(); + assert_eq!( + rates, + [ + None, + Some(1_000_000.0 / 30.0 / 10_000.0), + None, + Some(1.0), + None + ] + ); + + let summary = summarize(&samples, "node").unwrap(); + assert_eq!(summary.restarts, 1); + assert_eq!(summary.readings, 5); + } + + #[test] + fn a_summary_averages_and_peaks_and_sums_what_moved() { + let samples = [ + sample(1_000, Some(reading(START, 0, 100, 0))), + sample(1_030, Some(reading(START, 3_000_000, 300, 6_000))), + sample(1_060, Some(reading(START, 9_000_000, 200, 9_000))), + ]; + let summary = summarize(&samples, "node").unwrap(); + assert_eq!(summary.cpu_avg_pct, Some(15.0)); + assert_eq!(summary.cpu_peak_pct, Some(20.0)); + assert_eq!(summary.mem_avg, 200); + assert_eq!(summary.mem_peak, 300); + assert_eq!(summary.net_rx, 9_000); + assert_eq!(summary.first_at, 1_000); + assert_eq!(summary.last_at, 1_060); + assert!(summarize(&samples, "absent").is_none()); + } + + #[test] + fn appended_samples_read_back_and_a_torn_line_is_skipped() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("lab.jsonl"); + append_at(&path, &sample(1_000, Some(reading(START, 1, 2, 3)))).unwrap(); + append_at(&path, &sample(1_030, Some(reading(START, 4, 5, 6)))).unwrap(); + // A crash mid-write leaves a partial line at the end. + let mut text = std::fs::read_to_string(&path).unwrap(); + text.push_str(r#"{"at":1060,"servi"#); + std::fs::write(&path, text).unwrap(); + + let samples = read_at(&path); + assert_eq!(samples.len(), 2); + assert_eq!(samples[1].services["node"].cpu_usec, 4); + } + + #[test] + fn a_tail_read_drops_the_partial_first_line() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("lab.jsonl"); + for at in 0..10 { + append_at( + &path, + &sample(1_000 + at * 30, Some(reading(START, at, 0, 0))), + ) + .unwrap(); + } + let whole = std::fs::metadata(&path).unwrap().len(); + let line = whole / 10; + + let tail = read_tail_at(&path, line * 3 + line / 2); + let times: Vec = tail.iter().map(|sample| sample.at).collect(); + assert_eq!(times, [1_210, 1_240, 1_270]); + assert_eq!(read_tail_at(&path, whole * 2).len(), 10); + } + + #[test] + fn pruning_keeps_only_the_retention_window() { + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("lab.jsonl"); + for at in [100, 200, 300, 400] { + append_at(&path, &sample(at, Some(reading(START, 0, 0, 0)))).unwrap(); + } + prune_at(&path, 450, 200).unwrap(); + let times: Vec = read_at(&path).iter().map(|sample| sample.at).collect(); + assert_eq!(times, [300, 400]); + assert!(!dir.path().join("lab.jsonl.tmp").exists()); + } + + #[test] + fn units_read_at_a_glance() { + assert_eq!(format_bytes(332_886_016), "317 MiB"); + assert_eq!(format_bytes(1 << 30), "1.0 GiB"); + assert_eq!(format_bytes(2_048), "2 KiB"); + assert_eq!(format_bytes(12), "12 B"); + assert_eq!(format_pct(0.74), "0.7%"); + assert_eq!(format_pct(37.4), "37%"); + } +} diff --git a/crates/supervisor/src/lib.rs b/crates/supervisor/src/lib.rs index 79db911..502ba9b 100644 --- a/crates/supervisor/src/lib.rs +++ b/crates/supervisor/src/lib.rs @@ -12,11 +12,15 @@ //! handlers and writes the health cache. //! - [`cache`] and [`lock`] are the two pieces of on-disk state: the health //! `ps` reads, and the pid that keeps two supervisors off one project. +//! - [`history`] is the third: the resource samples `stats` and `top` read, +//! written by a sampler thread `run` starts. pub mod cache; pub mod clock; +pub mod history; pub mod lock; pub mod run; +mod sampler; pub mod tick; pub use run::{interrupt_action, run, OnInterrupt, Options, SupervisorError}; diff --git a/crates/supervisor/src/run.rs b/crates/supervisor/src/run.rs index 17ba8d5..6b3fdbf 100644 --- a/crates/supervisor/src/run.rs +++ b/crates/supervisor/src/run.rs @@ -8,7 +8,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::io::Write; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Arc; +use std::sync::{mpsc, Arc}; use std::time::{Duration, Instant}; use signal_hook::consts::{SIGINT, SIGTERM}; @@ -18,6 +18,8 @@ use container_compose_stack::resolve::{ResolvedHealthcheck, ResolvedStack}; use crate::cache::{self, CachedHealth, HealthCache}; use crate::clock; +use crate::history; +use crate::sampler::Sampler; use crate::tick::{Action, ProbeOutcome, ServiceSpec, Supervisor, Update, View}; #[derive(Debug, Clone)] @@ -30,6 +32,8 @@ pub struct Options { /// The first interrupt leaves everything running. pub no_stop_on_exit: bool, pub verbose: bool, + /// How often to record resource usage to the history; `None` records none. + pub sample_interval: Option, } #[derive(Debug)] @@ -103,7 +107,25 @@ pub fn run( .collect(), ); let start = Instant::now(); - let mut history: BTreeMap = BTreeMap::new(); + let mut records: BTreeMap = BTreeMap::new(); + + let (warnings_tx, warnings) = mpsc::channel::(); + if let (Some(interval), Some(path)) = (options.sample_interval, history::path(&stack.name)) { + Sampler { + runtime: options.runtime.clone(), + path, + containers: services + .iter() + .map(|service| (argv::container_name(&stack.name, service), service.clone())) + .collect(), + interval, + interrupts: interrupts.clone(), + warnings: warnings_tx, + } + // Not joined: it checks the interrupt counter every 100 ms, and a + // sample it appends while the services stop is harmless. + .spawn(); + } writeln!( out, @@ -111,12 +133,15 @@ pub fn run( services.len(), if services.len() == 1 { "" } else { "s" } )?; - publish(stack, &supervisor, &history, out); + publish(stack, &supervisor, &records, out); loop { if interrupted(&interrupts) { break; } + while let Ok(warning) = warnings.try_recv() { + let _ = writeln!(out, "warning: {warning}"); + } let now_ms = elapsed(start); let containers = match observe::containers(&options.runtime) { @@ -157,13 +182,13 @@ pub fn run( "give up {service} after {attempts} restart{}", if attempts == 1 { "" } else { "s" } ); - history.entry(service).or_default().gave_up = true; + records.entry(service).or_default().gave_up = true; cache_dirty = true; } Action::Restart { service, attempt } => { let name = argv::container_name(&stack.name, &service); let _ = writeln!(out, "restart {name} attempt {attempt}"); - let record = history.entry(service).or_default(); + let record = records.entry(service).or_default(); record.restarts = record.restarts.saturating_add(1); record.last_restart_at = Some(clock::format_rfc3339_seconds(clock::wall_ms() / 1_000)); @@ -200,7 +225,7 @@ pub fn run( } if cache_dirty { - publish(stack, &supervisor, &history, out); + publish(stack, &supervisor, &records, out); } sleep(options.poll_interval, &interrupts); } @@ -281,7 +306,7 @@ fn views_for( fn publish( stack: &ResolvedStack, supervisor: &Supervisor, - history: &BTreeMap, + records: &BTreeMap, out: &mut dyn Write, ) { let services = supervisor @@ -291,7 +316,7 @@ fn publish( let checked_at = supervisor .checked_at(&name) .map(|millis| clock::format_rfc3339_seconds(millis / 1_000)); - let record = history.get(&name).cloned().unwrap_or_default(); + let record = records.get(&name).cloned().unwrap_or_default(); ( name, CachedHealth { diff --git a/crates/supervisor/src/sampler.rs b/crates/supervisor/src/sampler.rs new file mode 100644 index 0000000..74c85c8 --- /dev/null +++ b/crates/supervisor/src/sampler.rs @@ -0,0 +1,198 @@ +//! The sampler: a thread that records the stack's resource usage while the +//! supervisor runs. +//! +//! It is a thread rather than a step of the supervisor's loop because +//! `container stats --no-stream` blocks for two seconds, which would delay +//! every restart and probe behind it. It owns its state and shares only the +//! interrupt counter; anything worth saying goes back to the loop, which owns +//! the output, as a message. + +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::Sender; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use container_compose_runtime::{observe, ObservedContainer, ObservedStats}; + +use crate::clock; +use crate::history::{self, Reading, Sample}; + +/// Pruning rewrites the file, so it runs at start and then hourly, not per sample. +const PRUNE_EVERY: Duration = Duration::from_secs(3_600); + +pub(crate) struct Sampler { + pub runtime: String, + pub path: PathBuf, + /// Container name to service name, for the services being supervised. + pub containers: BTreeMap, + pub interval: Duration, + pub interrupts: Arc, + pub warnings: Sender, +} + +impl Sampler { + pub fn spawn(self) -> JoinHandle<()> { + thread::spawn(move || self.work()) + } + + fn work(self) { + let mut last_warning: Option = None; + let mut pruned_at: Option = None; + // The first sample waits one interval: the start phase is the supervisor's + // busiest moment, and a reading from the first seconds of a container + // says little. It also keeps the sampler's `container list` out of short + // test runs, where the fake runtime's world file has a single writer. + while self.wait() { + if pruned_at.is_none_or(|at| at.elapsed() >= PRUNE_EVERY) { + let now = clock::wall_ms() / 1_000; + if let Err(error) = history::prune_at(&self.path, now, history::RETENTION_SECS) { + if error.kind() != std::io::ErrorKind::NotFound { + self.warn( + &mut last_warning, + format!("cannot prune the history: {error}"), + ); + } + } + pruned_at = Some(Instant::now()); + } + + match self.sample() { + Ok(Some(sample)) => match history::append_at(&self.path, &sample) { + Ok(()) => last_warning = None, + Err(error) => { + self.warn( + &mut last_warning, + format!("cannot write the history: {error}"), + ); + } + }, + Ok(None) => {} + Err(error) => self.warn(&mut last_warning, format!("cannot sample usage: {error}")), + } + } + } + + /// Sleep one interval; `false` when an interrupt arrived meanwhile. + fn wait(&self) -> bool { + let deadline = Instant::now() + self.interval; + while Instant::now() < deadline { + if self.interrupts.load(Ordering::SeqCst) > 0 { + return false; + } + thread::sleep(Duration::from_millis(100)); + } + self.interrupts.load(Ordering::SeqCst) == 0 + } + + /// One reading per supervised container the runtime reported on; `None` + /// when it reported on none of them, so the file records no empty lines. + fn sample(&self) -> Result, String> { + let stats = observe::stats(&self.runtime).map_err(|error| error.to_string())?; + let containers = observe::containers(&self.runtime).map_err(|error| error.to_string())?; + let sample = to_sample( + clock::wall_ms() / 1_000, + &self.containers, + &stats, + &containers, + ); + Ok((!sample.services.is_empty()).then_some(sample)) + } + + /// Say each distinct problem once, not every 30 seconds. + fn warn(&self, last: &mut Option, message: String) { + if last.as_deref() != Some(message.as_str()) { + let _ = self.warnings.send(message.clone()); + *last = Some(message); + } + } +} + +fn to_sample( + at: u64, + names: &BTreeMap, + stats: &[ObservedStats], + containers: &[ObservedContainer], +) -> Sample { + let services = stats + .iter() + .filter_map(|reading| { + let service = names.get(&reading.id)?; + let started = containers + .iter() + .find(|container| container.id == reading.id) + .and_then(|container| container.status.started_date.clone()); + Some(( + service.clone(), + Reading { + started, + cpu_usec: reading.cpu_usage_usec, + mem: reading.memory_usage_bytes, + mem_limit: reading.memory_limit_bytes, + net_rx: reading.network_rx_bytes, + net_tx: reading.network_tx_bytes, + blk_read: reading.block_read_bytes, + blk_write: reading.block_write_bytes, + pids: reading.num_processes, + }, + )) + }) + .collect(); + Sample { at, services } +} + +#[cfg(test)] +mod tests { + use super::to_sample; + use container_compose_runtime::{ObservedContainer, ObservedStats}; + use std::collections::BTreeMap; + + fn fixture(name: &str) -> String { + std::fs::read_to_string(format!( + "{}/../../fixtures/real/{name}", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() + } + + #[test] + fn only_supervised_containers_become_readings() { + let stats: Vec = + serde_json::from_str(&fixture("container-stats.json")).unwrap(); + let names = BTreeMap::from([ + ("midnight-lab-node".to_owned(), "node".to_owned()), + ("midnight-lab-absent".to_owned(), "absent".to_owned()), + ]); + let sample = to_sample(1_000, &names, &stats, &[]); + + let services: Vec<&str> = sample.services.keys().map(String::as_str).collect(); + assert_eq!( + services, + ["node"], + "unsupervised and unreported are left out" + ); + let node = &sample.services["node"]; + assert_eq!(node.cpu_usec, 6_532_864); + assert_eq!(node.mem, 332_886_016); + assert_eq!(node.started, None, "no container listed, so no start time"); + } + + #[test] + fn the_start_time_comes_from_the_container_list() { + let stats: Vec = + serde_json::from_str(&fixture("container-stats.json")).unwrap(); + // The captured list belongs to a different capture; reuse its `api` + // container's shape under the node's name. + let mut containers: Vec = + serde_json::from_str(&fixture("container-list.json")).unwrap(); + containers.retain(|container| container.status.started_date.is_some()); + containers[0].id = "midnight-lab-node".to_owned(); + let started = containers[0].status.started_date.clone(); + + let names = BTreeMap::from([("midnight-lab-node".to_owned(), "node".to_owned())]); + let sample = to_sample(1_000, &names, &stats, &containers); + assert_eq!(sample.services["node"].started, started); + } +} diff --git a/crates/top/src/app.rs b/crates/top/src/app.rs index 8267d59..2e5828a 100644 --- a/crates/top/src/app.rs +++ b/crates/top/src/app.rs @@ -8,6 +8,8 @@ use std::collections::VecDeque; use ratatui::crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use ratatui::widgets::TableState; +use container_compose_supervisor::history::Sample; + use crate::model::{self, Declared, Event, Row}; use crate::poll::{Logs, Request, Snapshot, Update}; @@ -29,6 +31,8 @@ pub struct App { pub updated_at: Option, /// The last on-demand probe, shown in the footer. pub note: Option, + /// The recent resource history, as last read. + pub samples: Vec, pub quit: bool, polled: bool, } @@ -46,6 +50,7 @@ impl App { supervisor_pid: None, updated_at: None, note: None, + samples: Vec::new(), quit: false, polled: false, } @@ -72,6 +77,9 @@ impl App { let before = self.selected_service().map(str::to_owned); self.updated_at = Some(model::clock_of_day(snapshot.at_secs)); self.supervisor_pid = snapshot.cache.as_ref().map(|cache| cache.pid); + if let Some(samples) = snapshot.history { + self.samples = samples; + } match snapshot.containers { Ok(containers) => { @@ -80,6 +88,7 @@ impl App { &self.declared, &containers, snapshot.cache.as_ref(), + &self.samples, snapshot.at_secs, ); // The first poll is the baseline, not a burst of "appeared". @@ -196,6 +205,7 @@ mod tests { containers: Ok(containers), cache: None, logs, + history: None, }) } @@ -235,6 +245,7 @@ mod tests { containers: Err("`container list` exited with 1".to_owned()), cache: None, logs: None, + history: None, })); assert_eq!(app.rows.len(), 2); assert!(app.runtime_error.is_some()); diff --git a/crates/top/src/lib.rs b/crates/top/src/lib.rs index bf26ce2..cb30dbf 100644 --- a/crates/top/src/lib.rs +++ b/crates/top/src/lib.rs @@ -78,6 +78,7 @@ pub fn run(options: Options) -> io::Result<()> { healthchecks, interval: options.interval, log_lines: options.log_lines, + history: container_compose_supervisor::history::path(&declared.project), }); let mut app = App::new(declared); diff --git a/crates/top/src/model.rs b/crates/top/src/model.rs index fa5ac6b..9b36f78 100644 --- a/crates/top/src/model.rs +++ b/crates/top/src/model.rs @@ -10,6 +10,7 @@ use container_compose_runtime::ObservedContainer; use container_compose_stack::resolve::ResolvedStack; use container_compose_supervisor::cache::HealthCache; use container_compose_supervisor::clock; +use container_compose_supervisor::history::{self, Point, Sample}; /// What the stack file declares. `top` runs without one, as `ps` does, so an /// interrupted session stays inspectable; then only the project is known. @@ -81,6 +82,98 @@ pub struct Row { pub policy: Option, pub ports: Vec, pub image: Option, + pub usage: Usage, +} + +/// A service's resource use, from the supervisor's history. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Usage { + /// CPU over the last sampled interval, in percent of one core; `None` + /// without a recent reading. + pub cpu: Option, + /// Memory at the last reading, with the limit. + pub mem: Option, + /// One character per [`TREND_BUCKET_SECS`] over the last + /// [`TREND_WINDOW_SECS`], oldest first; a space where there is no reading. + pub cpu_trend: String, + pub mem_trend: String, +} + +/// The span a trend covers. +pub const TREND_WINDOW_SECS: u64 = 3_600; +/// The span one trend character covers: twelve characters for the hour. +pub const TREND_BUCKET_SECS: u64 = 300; +/// A reading older than this is not "now": three missed samples. +const CURRENT_WITHIN_SECS: u64 = 90; +/// A CPU trend is scaled to its own peak, but never below this, so an idle +/// service's noise draws as a flat floor rather than full-height bars. +const CPU_TREND_FLOOR_PCT: f64 = 5.0; +const BARS: [char; 8] = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']; + +pub fn usage(samples: &[Sample], service: &str, running: bool, now_secs: u64) -> Usage { + let points = history::series(samples, service); + let since = now_secs.saturating_sub(TREND_WINDOW_SECS); + let window: Vec<&Point> = points.iter().filter(|point| point.at > since).collect(); + + let current = points + .last() + .filter(|point| running && now_secs.saturating_sub(point.at) <= CURRENT_WITHIN_SECS); + + let cpu_peak = window + .iter() + .filter_map(|point| point.cpu_pct) + .fold(CPU_TREND_FLOOR_PCT, f64::max); + let mem_scale = window + .iter() + .map(|point| point.mem_limit) + .max() + .filter(|limit| *limit > 0) + .or_else(|| window.iter().map(|point| point.mem).max()) + .unwrap_or(1) as f64; + + Usage { + cpu: current + .and_then(|point| point.cpu_pct) + .map(history::format_pct), + mem: current.map(|point| { + if point.mem_limit > 0 { + format!( + "{} / {}", + history::format_bytes(point.mem), + history::format_bytes(point.mem_limit) + ) + } else { + history::format_bytes(point.mem) + } + }), + cpu_trend: trend(&window, since, |point| point.cpu_pct, cpu_peak), + mem_trend: trend(&window, since, |point| Some(point.mem as f64), mem_scale), + } +} + +/// Average each bucket's values and draw it as one bar, scaled to `scale`. +fn trend(points: &[&Point], since: u64, value: fn(&Point) -> Option, scale: f64) -> String { + let buckets = usize::try_from(TREND_WINDOW_SECS / TREND_BUCKET_SECS).unwrap_or(12); + let mut sums = vec![(0.0_f64, 0_u32); buckets]; + for point in points { + let Some(value) = value(point) else { continue }; + let offset = point.at.saturating_sub(since).saturating_sub(1); + let index = usize::try_from(offset / TREND_BUCKET_SECS) + .unwrap_or(usize::MAX) + .min(buckets - 1); + sums[index].0 += value; + sums[index].1 += 1; + } + sums.iter() + .map(|(sum, count)| { + if *count == 0 { + return ' '; + } + let fraction = (sum / f64::from(*count) / scale).clamp(0.0, 1.0); + let level = (fraction * (BARS.len() - 1) as f64).round() as usize; + BARS[level] + }) + .collect() } impl Row { @@ -91,15 +184,22 @@ impl Row { /// One row per declared service, in start order, then one per undeclared /// container of the project. Only a cache that [`cache::live`] trusted -/// should be passed: health and restarts come from it alone. +/// should be passed: health and restarts come from it alone. Usage comes from +/// `samples`, the recent history. /// /// [`cache::live`]: container_compose_supervisor::cache::live pub fn rows( declared: &Declared, containers: &[ObservedContainer], cache: Option<&HealthCache>, + samples: &[Sample], now_secs: u64, ) -> Vec { + let context = Context { + cache, + samples, + now_secs, + }; let mine: Vec<&ObservedContainer> = containers .iter() .filter(|container| container.project() == Some(declared.project.as_str())) @@ -113,13 +213,7 @@ pub fn rows( .iter() .copied() .find(|container| container.service() == Some(service.as_str())); - row( - service, - container, - declared.services.get(service), - cache, - now_secs, - ) + row(service, container, declared.services.get(service), &context) }) .collect(); @@ -133,7 +227,7 @@ pub fn rows( }) .map(|container| { let service = container.service().unwrap_or(&container.id).to_owned(); - row(&service, Some(container), None, cache, now_secs) + row(&service, Some(container), None, &context) }) .collect(); undeclared.sort_by(|a, b| a.service.cmp(&b.service)); @@ -141,14 +235,21 @@ pub fn rows( rows } +/// What every row is computed against. +struct Context<'a> { + cache: Option<&'a HealthCache>, + samples: &'a [Sample], + now_secs: u64, +} + fn row( service: &str, container: Option<&ObservedContainer>, declared: Option<&DeclaredService>, - cache: Option<&HealthCache>, - now_secs: u64, + context: &Context, ) -> Row { - let cached = cache.and_then(|cache| cache.services.get(service)); + let now_secs = context.now_secs; + let cached = context.cache.and_then(|cache| cache.services.get(service)); let running = container.is_some_and(ObservedContainer::is_running); let health = if !running { @@ -181,6 +282,7 @@ fn row( .map(|service| service.ports.clone()) .unwrap_or_default(), image: container.map(|container| container.configuration.image.reference.clone()), + usage: usage(context.samples, service, running, now_secs), } } @@ -263,11 +365,13 @@ pub fn clock_of_day(seconds: u64) -> String { #[cfg(test)] mod tests { use super::{ - events, format_span, rows, Declared, DeclaredService, Row, NOT_CREATED, NO_CHECK, NO_HEALTH, + events, format_span, rows, usage, Declared, DeclaredService, Row, Usage, NOT_CREATED, + NO_CHECK, NO_HEALTH, TREND_WINDOW_SECS, }; use container_compose_runtime::ObservedContainer; use container_compose_supervisor::cache::{CachedHealth, HealthCache}; use container_compose_supervisor::clock; + use container_compose_supervisor::history::{Reading, Sample}; use std::collections::BTreeMap; /// The `container list --all --format json` response captured from the @@ -337,7 +441,13 @@ mod tests { #[test] fn declared_services_come_first_in_start_order() { - let rows = rows(&declared(&["db"]), &captured(), None, started_at_of_api()); + let rows = rows( + &declared(&["db"]), + &captured(), + None, + &[], + started_at_of_api(), + ); let services: Vec<&str> = rows.iter().map(|row| row.service.as_str()).collect(); assert_eq!(services, ["api", "worker", "db"]); assert_eq!(rows[2].state, NOT_CREATED); @@ -349,7 +459,7 @@ mod tests { let declared = declared(&[]); let now = started_at_of_api() + 125; let cache = cache(&declared.project, healthy(2, false)); - let rows = rows(&declared, &captured(), Some(&cache), now); + let rows = rows(&declared, &captured(), Some(&cache), &[], now); assert_eq!(rows[0].state, "running"); assert_eq!(rows[0].health, "healthy"); @@ -364,12 +474,12 @@ mod tests { #[test] fn a_running_service_without_a_cache_is_unknown_unless_it_has_no_check() { let declared = declared(&[]); - let rows = rows(&declared, &captured(), None, started_at_of_api()); + let rows = rows(&declared, &captured(), None, &[], started_at_of_api()); assert_eq!(rows[0].health, "unknown"); let mut unchecked = declared.clone(); unchecked.services.get_mut("api").unwrap().has_healthcheck = false; - let rows = super::rows(&unchecked, &captured(), None, started_at_of_api()); + let rows = super::rows(&unchecked, &captured(), None, &[], started_at_of_api()); assert_eq!(rows[0].health, NO_CHECK); } @@ -380,6 +490,7 @@ mod tests { &Declared::bare(project), &captured(), None, + &[], started_at_of_api(), ); let services: Vec<&str> = rows.iter().map(|row| row.service.as_str()).collect(); @@ -393,6 +504,7 @@ mod tests { &Declared::bare("someone-else".to_owned()), &captured(), None, + &[], 0, ); assert!(rows.is_empty()); @@ -418,9 +530,76 @@ mod tests { policy: Some("on-stop".to_owned()), ports: Vec::new(), image: None, + usage: Usage::default(), + } + } + + fn reading(cpu_usec: u64, mem: u64) -> Reading { + Reading { + started: Some("2026-10-26T02:00:00Z".to_owned()), + cpu_usec, + mem, + mem_limit: 1 << 30, + ..Reading::default() } } + fn samples(readings: &[(u64, u64, u64)]) -> Vec { + readings + .iter() + .map(|(at, cpu_usec, mem)| Sample { + at: *at, + services: BTreeMap::from([("node".to_owned(), reading(*cpu_usec, *mem))]), + }) + .collect() + } + + #[test] + fn usage_shows_the_latest_interval_and_the_limit() { + // 3 s of CPU in 30 s is a tenth of a core. + let samples = samples(&[(10_000, 0, 100 << 20), (10_030, 3_000_000, 300 << 20)]); + let usage = usage(&samples, "node", true, 10_040); + assert_eq!(usage.cpu.as_deref(), Some("10%")); + assert_eq!(usage.mem.as_deref(), Some("300 MiB / 1.0 GiB")); + } + + #[test] + fn a_stale_or_stopped_service_has_no_current_usage() { + let samples = samples(&[(10_000, 0, 1), (10_030, 3_000_000, 1)]); + assert_eq!(usage(&samples, "node", true, 10_030 + 91).cpu, None); + assert_eq!(usage(&samples, "node", false, 10_031).mem, None); + assert_eq!( + usage(&[], "node", true, 10_031), + Usage { + cpu_trend: " ".repeat(12), + mem_trend: " ".repeat(12), + ..Usage::default() + } + ); + } + + #[test] + fn a_trend_is_twelve_five_minute_bars_with_gaps_left_blank() { + let now = 100_000; + let start = now - TREND_WINDOW_SECS; + // Busy (one core) in the first bucket, idle in the last; nothing between. + let samples = samples(&[ + (start + 10, 0, 512 << 20), + (start + 40, 30_000_000, 512 << 20), + (now - 60, 31_000_000, 1 << 30), + (now - 30, 31_000_000, 1 << 30), + ]); + let usage = usage(&samples, "node", true, now); + let cpu: Vec = usage.cpu_trend.chars().collect(); + let mem: Vec = usage.mem_trend.chars().collect(); + assert_eq!(cpu.len(), 12); + assert_eq!(cpu[0], '█', "a full core against a 100% peak"); + assert!(cpu[1..11].iter().all(|bar| *bar == ' '), "{cpu:?}"); + assert_eq!(cpu[11], '▁', "idle draws as the floor, not as nothing"); + assert_eq!(mem[0], '▅', "half the limit"); + assert_eq!(mem[11], '█', "the whole limit"); + } + #[test] fn events_name_state_health_restart_and_give_up_changes() { let before = base(); diff --git a/crates/top/src/poll.rs b/crates/top/src/poll.rs index 84d5b5f..b804f67 100644 --- a/crates/top/src/poll.rs +++ b/crates/top/src/poll.rs @@ -6,6 +6,7 @@ //! request sender ends the worker. use std::collections::BTreeMap; +use std::path::PathBuf; use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; use std::thread; use std::time::{Duration, Instant}; @@ -13,6 +14,7 @@ use std::time::{Duration, Instant}; use container_compose_runtime::{argv, observe, probe, Command, ObservedContainer, Probe}; use container_compose_supervisor::cache::{self, HealthCache}; use container_compose_supervisor::clock; +use container_compose_supervisor::history::{self, Sample}; /// Where to look, and how often. #[derive(Debug, Clone)] @@ -25,8 +27,17 @@ pub struct Source { pub interval: Duration, /// Log lines fetched for the selected service. pub log_lines: u32, + /// The supervisor's resource history for this project, if there is a home + /// directory to find it in. + pub history: Option, } +/// The history changes every 30 seconds, so re-reading it every poll is waste. +const HISTORY_EVERY: Duration = Duration::from_secs(10); +/// Enough of the file's end for the trend window: an hour of 30-second samples +/// is 120 lines, a few hundred bytes each for a handful of services. +const HISTORY_TAIL_BYTES: u64 = 512 * 1024; + #[derive(Debug, Clone, PartialEq, Eq)] pub enum Request { /// The service whose logs to fetch; `None` fetches none. Polls at once. @@ -50,6 +61,8 @@ pub struct Snapshot { /// The health cache, only when [`cache::live`] trusts it. pub cache: Option, pub logs: Option, + /// The recent history, when this poll re-read it; `None` keeps the last. + pub history: Option>, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -69,6 +82,7 @@ pub fn spawn(source: Source) -> (Sender, Receiver) { fn work(source: &Source, inbox: &Receiver, outbox: &Sender) { let mut selected: Option = None; let mut due = Instant::now(); + let mut history_read: Option = None; loop { match inbox.recv_timeout(due.saturating_duration_since(Instant::now())) { Ok(Request::Select(service)) => { @@ -91,7 +105,17 @@ fn work(source: &Source, inbox: &Receiver, outbox: &Sender) { Err(RecvTimeoutError::Disconnected) => return, } - let snapshot = capture(source, selected.as_deref()); + let mut snapshot = capture(source, selected.as_deref()); + if history_read.is_none_or(|at| at.elapsed() >= HISTORY_EVERY) { + snapshot.history = Some( + source + .history + .as_deref() + .map(|path| history::read_tail_at(path, HISTORY_TAIL_BYTES)) + .unwrap_or_default(), + ); + history_read = Some(Instant::now()); + } if outbox.send(Update::Snapshot(snapshot)).is_err() { return; } @@ -120,6 +144,7 @@ fn capture(source: &Source, selected: Option<&str>) -> Snapshot { containers, cache, logs, + history: None, } } diff --git a/crates/top/src/ui.rs b/crates/top/src/ui.rs index d780257..2405d04 100644 --- a/crates/top/src/ui.rs +++ b/crates/top/src/ui.rs @@ -53,7 +53,8 @@ fn draw_header(frame: &mut Frame, area: Rect, app: &App) { fn draw_services(frame: &mut Frame, area: Rect, app: &mut App) { let header = TableRow::new([ - "SERVICE", "STATE", "HEALTH", "UPTIME", "RESTARTS", "POLICY", "PORTS", "IMAGE", + "SERVICE", "STATE", "HEALTH", "UPTIME", "RESTARTS", "CPU", "CPU 1h", "MEMORY", "MEM 1h", + "PORTS", ]) .style(Style::new().bold()); @@ -61,13 +62,16 @@ fn draw_services(frame: &mut Frame, area: Rect, app: &mut App) { let table = Table::new( rows, [ - Constraint::Length(14), + Constraint::Length(12), Constraint::Length(11), - Constraint::Length(10), + Constraint::Length(9), Constraint::Length(8), - Constraint::Length(10), Constraint::Length(8), - Constraint::Length(24), + Constraint::Length(6), + Constraint::Length(12), + // `317 MiB / 1.0 GiB` + Constraint::Length(17), + Constraint::Length(12), Constraint::Fill(1), ], ) @@ -112,15 +116,18 @@ fn service_row(row: &Row) -> TableRow<'static> { (0, false) => Cell::from("0").dark_gray(), (count, false) => Cell::from(count.to_string()).yellow(), }; + let dash = || "—".to_owned(); TableRow::new([ Cell::from(row.service.clone()).bold(), Cell::from(row.state.clone()).style(state_style), Cell::from(row.health.clone()).style(health_style), - Cell::from(row.uptime.clone().unwrap_or_else(|| "—".to_owned())), + Cell::from(row.uptime.clone().unwrap_or_else(dash)), restarts, - Cell::from(row.policy.clone().unwrap_or_else(|| "—".to_owned())), + Cell::from(row.usage.cpu.clone().unwrap_or_else(dash)), + Cell::from(row.usage.cpu_trend.clone()).cyan(), + Cell::from(row.usage.mem.clone().unwrap_or_else(dash)), + Cell::from(row.usage.mem_trend.clone()).magenta(), Cell::from(row.ports.join(", ")), - Cell::from(row.image.clone().unwrap_or_default()).dark_gray(), ]) } @@ -148,15 +155,29 @@ fn draw_events(frame: &mut Frame, area: Rect, app: &App) { fn draw_logs(frame: &mut Frame, area: Rect, app: &App) { let service = app.selected_service().unwrap_or(""); + let row = app.rows.iter().find(|row| row.service == service); + // The table has no room for these; the selected service's pane does. + let details: String = row + .into_iter() + .flat_map(|row| { + [ + row.image.clone(), + row.policy + .as_ref() + .map(|policy| format!("restart {policy}")), + ] + }) + .flatten() + .map(|detail| format!(" · {detail}")) + .collect(); let scrolled = if app.log_scroll > 0 { format!(" · {} lines up", app.log_scroll) } else { String::new() }; - let block = Block::bordered().title(format!(" logs · {service}{scrolled} ")); + let block = Block::bordered().title(format!(" logs · {service}{details}{scrolled} ")); let Some(logs) = &app.logs else { - let row = app.rows.iter().find(|row| row.service == service); let message = match row { Some(row) if row.container.is_none() => "the container has not been created", Some(_) => "fetching…", @@ -199,8 +220,10 @@ mod tests { use crate::model::Declared; use crate::poll::{Logs, Snapshot, Update}; use container_compose_runtime::ObservedContainer; + use container_compose_supervisor::history::{Reading, Sample}; use ratatui::backend::TestBackend; use ratatui::Terminal; + use std::collections::BTreeMap; fn captured() -> Vec { let text = std::fs::read_to_string(concat!( @@ -237,6 +260,7 @@ mod tests { containers: Ok(captured()), cache: None, logs: None, + history: None, })); app.apply(Update::Snapshot(Snapshot { at_secs: 1_792_830_353, @@ -247,10 +271,30 @@ mod tests { lines: vec!["capture-hello".to_owned()], error: None, }), + history: Some(history_for_api(1_792_830_353)), })); app } + /// Two readings of `api`, 30 s apart, ending just before `now`: 3 s of + /// CPU in the interval, 64 MiB of a 1 GiB limit. + fn history_for_api(now: u64) -> Vec { + let reading = |cpu_usec: u64| Reading { + started: Some("2026-10-24T04:25:54Z".to_owned()), + cpu_usec, + mem: 64 << 20, + mem_limit: 1 << 30, + ..Reading::default() + }; + [(now - 40, 1_000_000), (now - 10, 4_000_000)] + .into_iter() + .map(|(at, cpu_usec)| Sample { + at, + services: BTreeMap::from([("api".to_owned(), reading(cpu_usec))]), + }) + .collect() + } + #[test] fn the_screen_shows_services_events_and_the_selected_services_logs() { let mut app = app_with_logs(); @@ -268,6 +312,26 @@ mod tests { assert!(text.contains("q quit")); } + #[test] + fn usage_from_the_history_fills_the_cpu_and_memory_columns() { + let mut app = app_with_logs(); + let text = screen(&mut app, 130, 24); + println!("{text}"); + + let api = text + .lines() + .find(|line| line.contains("▶ api")) + .expect("the api row"); + assert!(api.contains("10%"), "3 s of CPU in 30 s: {api}"); + assert!(api.contains("64 MiB / 1.0 GiB"), "{api}"); + let worker = text.lines().find(|line| line.contains("worker")).unwrap(); + assert!(!worker.contains("MiB"), "no readings, no usage: {worker}"); + assert!( + text.contains("docker.io/library/alpine:latest"), + "image in the logs title" + ); + } + #[test] fn a_narrow_terminal_still_draws() { let mut app = app_with_logs(); diff --git a/dev/fake-container/src/main.rs b/dev/fake-container/src/main.rs index c8ff0fe..f207309 100644 --- a/dev/fake-container/src/main.rs +++ b/dev/fake-container/src/main.rs @@ -30,7 +30,7 @@ use serde::{Deserialize, Serialize}; use container_compose_runtime::{ ContainerConfiguration, ContainerImage, ContainerStatus, ImageConfiguration, NetworkConfiguration, NetworkStatus, Observed, ObservedContainer, ObservedImage, - ObservedNetwork, ObservedVolume, VolumeConfiguration, + ObservedNetwork, ObservedStats, ObservedVolume, VolumeConfiguration, }; #[derive(Debug, Default, Serialize, Deserialize)] @@ -169,6 +169,7 @@ fn dispatch(args: &[String], world: &mut World) -> Result<(), Failure> { "delete" | "rm" => delete(rest, world), "inspect" => inspect(rest, world), "logs" => logs(rest, world), + "stats" => stats(rest, world), "exec" => exec(rest, world), "volume" => match rest.first().map(String::as_str) { Some("create") => volume_create(&rest[1..], world), @@ -243,6 +244,38 @@ fn list(args: &[String], world: &mut World) -> Result<(), Failure> { Ok(()) } +/// One record per running container, in the real response's shape. The fake +/// runs no processes, so every counter is zero rather than invented; only the +/// set of containers reported is meaningful. Like the real runtime, naming a +/// container that is not running fails. +fn stats(args: &[String], world: &mut World) -> Result<(), Failure> { + let parsed = parse(args, &["--format"], &["--no-stream", "--debug"])?; + for name in &parsed.positionals { + let running = world + .observed + .container(name) + .is_some_and(ObservedContainer::is_running); + if !running { + return Err(Failure::domain(format!("container {name} is not running"))); + } + } + let output: Vec = world + .observed + .containers + .iter() + .filter(|container| container.is_running()) + .filter(|container| { + parsed.positionals.is_empty() || parsed.positionals.contains(&container.id) + }) + .map(|container| ObservedStats { + id: container.id.clone(), + ..ObservedStats::default() + }) + .collect(); + println!("{}", serde_json::to_string(&output).expect("json")); + Ok(()) +} + fn create(args: &[String], world: &mut World) -> Result<(), Failure> { let (parsed, _command) = parse_until_positional( args, diff --git a/docs/01-model.md b/docs/01-model.md index 333bdba..e1c6a5c 100644 --- a/docs/01-model.md +++ b/docs/01-model.md @@ -218,7 +218,16 @@ cache — `~/Library/Caches/container-compose/.json`, holding its pid a the last result per service, with the restarts it has issued for each and whether it gave up — and `ps` and `top` read it. The restart count is history for a reader and never resets; the tick's attempt counter, which does reset after 60 s -of healthy uptime, stays in memory. A missing file, a pid that is +of healthy uptime, stays in memory. + +Resource use is the other thing only a running supervisor sees over time. A +sampler thread — a thread because `container stats` blocks for two seconds — +appends the runtime's raw counters every 30 s to a per-project history under +`~/Library/Application Support/container-compose/history/`, kept for seven days. +Unlike the health cache, it is not disposable: it outlives the supervisor that +wrote it, so `stats` can summarise yesterday. The sampler shares only the +interrupt counter with the loop and reports problems to it over a channel, each +distinct one once. A missing file, a pid that is gone, or a config-hash mismatch means `unknown`, never a stale "healthy". The cache is never authority: `up` readiness uses in-process state, deleting the file changes only what `ps` prints, and failing to write it warns rather than diff --git a/docs/03-cli.md b/docs/03-cli.md index 1f24305..2d2dfc1 100644 --- a/docs/03-cli.md +++ b/docs/03-cli.md @@ -220,15 +220,23 @@ container-compose top [--interval 1s] [-n 200] including any not yet created — then any undeclared container of the project: ``` - midnight-lab supervisor pid 5630 · updated 01:50:28Z - SERVICE STATE HEALTH UPTIME RESTARTS POLICY PORTS -▶ node running healthy 32s 0 no 127.0.0.1:9944->9944 - indexer running no check 12s 1 on-stop 127.0.0.1:8088->8088 + midnight-lab supervisor pid 96974 · updated 02:33:12Z + SERVICE STATE HEALTH UPTIME RESTARTS CPU CPU 1h MEMORY MEM 1h PORTS +▶ node running healthy 2m 00s 0 0.9% ▁▂ 312 MiB / 1.0 GiB ▃▃ 127.0.0.1:9944->9944 + indexer running no check 1m 54s 0 0.5% ▁▂ 158 MiB / 1.0 GiB ▂▂ 127.0.0.1:8088->8088 ``` +`CPU` and `MEMORY` are the last interval of the supervisor's resource history +(see [`stats`](#stats)): CPU in percent of one core, memory against the +container's limit. The `1h` columns are twelve five-minute bars, oldest first, +blank where there is no reading. CPU bars are scaled to the hour's peak (never +below 5%, so idle noise draws as a floor); memory bars to the limit. Without a +supervisor, or for a reading more than 90 s old, the values are `—`. + Below the table, an events pane lists what changed between polls — a state, a health, a restart, a give-up — newest first, and a logs pane follows the -selected service's last `-n` lines. +selected service's last `-n` lines. Its title carries the selected service's +image and restart policy. | key | does | |---|---| @@ -254,6 +262,53 @@ Like `ps`, it works without the stack file, from the project's containers alone; then there is no on-demand probe. Without a terminal it refuses (exit 1) and points at `ps --json`. +## `stats` + +```sh +container-compose stats [--since 1h] [--json] +``` + +Resource use per service over a window, from the supervisor's history: + +``` +midnight-lab · last 10m · 3 samples from 2026-10-26T02:31:51Z to 2026-10-26T02:32:56Z + +SERVICE READINGS CPU AVG CPU PEAK MEM AVG MEM PEAK LIMIT NET IN NET OUT DISK READ DISK WRITE RESTARTS +node 3 0.9% 1.0% 310 MiB 312 MiB 1.0 GiB 286 KiB 114 KiB 1 MiB 404 KiB 0 +``` + +While `up` supervises, a sampler thread runs `container stats` every 30 s and +appends one line per sample to +`~/Library/Application Support/container-compose/history/.jsonl` +(`CONTAINER_COMPOSE_HISTORY_DIR` overrides the directory). Lines older than +seven days are pruned at start and hourly; the midnight lab's three services +add about 1 MB a day. The file holds the runtime's raw counters — CPU time, +memory and its limit, network and block bytes, processes — and the container's +start time; rates are derived when read. + +- CPU is in percent of one core, so four busy cores is 400%. +- A container that restarted resets its counters. A changed start time breaks + the series there instead of producing a negative rate, and `RESTARTS` counts + those breaks. +- The bytes columns sum what moved over continuous intervals within the window. +- A gap — two samples more than 60 s apart — is time no supervisor ran, and the + header counts them. A service missing from a sample means the runtime gave + no reading for it; it is never counted as zero. +- The first sample is taken one interval after `up` starts supervising. +- `--since` takes the stack file's durations (`90m`, `168h`); there is no day + unit. + +The stack file only orders the rows; without one, services come from the +history. `--json` emits one document: `samples`, `first_at`, `last_at`, `gaps` +and a typed summary per service. + +The runtime's own `container stats` leaves out any container whose statistics +call fails, without saying so. One cause, found on the midnight-lab node: a +container whose PID 1 holds an auto-bound route netlink socket (libp2p's +interface watcher does) takes port ID 1, which the VM's init needs for its own +statistics socket. `init = true` on the service moves its process off PID 1; +apple/containerization#910 fixes the bind itself. + ## `logs` ```sh diff --git a/fixtures/real/capture.txt b/fixtures/real/capture.txt index 099f1bb..87428da 100644 --- a/fixtures/real/capture.txt +++ b/fixtures/real/capture.txt @@ -12,3 +12,5 @@ # container network list --format json -> network-list.json # container image list --format json -> image-list.json # container logs ccfixtures-31236-api -> logs-api.txt +# container stats --no-stream --format json -> container-stats.json (captured separately +# 2026-10-26 from the midnight-lab devnet, three running containers; container 1.6.0) diff --git a/fixtures/real/container-stats.json b/fixtures/real/container-stats.json new file mode 100644 index 0000000..2124741 --- /dev/null +++ b/fixtures/real/container-stats.json @@ -0,0 +1 @@ +[{"blockReadBytes":150818816,"blockWriteBytes":2552659968,"cpuUsageUsec":6532864,"id":"midnight-lab-node","memoryLimitBytes":1073741824,"memoryUsageBytes":332886016,"networkRxBytes":479307,"networkTxBytes":2025146,"numProcesses":43},{"blockReadBytes":35868672,"blockWriteBytes":34193408,"cpuUsageUsec":359157,"id":"midnight-lab-proof","memoryLimitBytes":1073741824,"memoryUsageBytes":77221888,"networkRxBytes":36475696,"networkTxBytes":507075,"numProcesses":6},{"blockReadBytes":142589952,"blockWriteBytes":4890624,"cpuUsageUsec":2464854,"id":"midnight-lab-indexer","memoryLimitBytes":1073741824,"memoryUsageBytes":169123840,"networkRxBytes":2275687,"networkTxBytes":189949,"numProcesses":19}] diff --git a/scripts/capture-runtime-fixtures.sh b/scripts/capture-runtime-fixtures.sh index e7ca66c..6eca846 100755 --- a/scripts/capture-runtime-fixtures.sh +++ b/scripts/capture-runtime-fixtures.sh @@ -103,6 +103,9 @@ done "$CONTAINER" network list --format json >"$OUT/network-list.json" "$CONTAINER" image list --format json >"$OUT/image-list.json" "$CONTAINER" logs "$API" >"$OUT/logs-api.txt" +# Takes two samples two seconds apart before printing. Only running containers +# appear, and a container whose statistics call fails is silently left out. +"$CONTAINER" stats --no-stream --format json "$API" >"$OUT/container-stats.json" { printf '# Captured %s by scripts/capture-runtime-fixtures.sh\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" @@ -118,6 +121,7 @@ done printf '# container network list --format json -> network-list.json\n' printf '# container image list --format json -> image-list.json\n' printf '# container logs %s -> logs-api.txt\n' "$API" + printf '# container stats --no-stream --format json %s -> container-stats.json\n' "$API" } >"$OUT/capture.txt" printf 'captured into %s\n' "$OUT" From bfed15f9d39a426ea2052989958c8e289c640aff Mon Sep 17 00:00:00 2001 From: Damon Oehlman Date: Sun, 1 Nov 2026 14:01:22 +1000 Subject: [PATCH 4/4] fix observe --- CLAUDE.md | 68 ++++++++++++++++++++++++++++++++--- crates/runtime/src/observe.rs | 37 +++++++++++-------- flake.lock | 12 +++---- 3 files changed, 91 insertions(+), 26 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d53624f..4b99019 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -399,9 +399,10 @@ nix develop -c cargo clippy --fix --allow-dirty --all-targets --all-features -- - Reading code with `rg`, `grep`, `sed`, `cat`, `head` or `wc` → use `at-peek`. Reading installed crate source or build-script output with `grep`, `sed` or `find` under `~/.cargo` or `target/…/build` → use `at-crate`. Reading history or working-tree state with `git status`, - `git diff` or `git log` → use `at-recall`. All three are read-only, bounded by construction, and - say what they did not show. `git show` and `git blame` have no `at-recall` verb yet — say so - rather than reaching for them silently. + `git diff` or `git log` → use `at-recall`. Asking the chain a question with `curl` → use + `at-cardano`. All four are read-only, bounded by construction, and say what they did not show. + `git show` and `git blame` have no `at-recall` verb yet — say so rather than reaching for them + silently. | Instead of | Use | | --- | --- | @@ -415,6 +416,7 @@ nix develop -c cargo clippy --fix --allow-dirty --all-targets --all-features -- | `git diff`, `git diff --stat` | `at-recall diff [] […]` — add `--patch` for the hunks | | `git log`, `git log --oneline` | `at-recall log [] […]` | | the archaeology before a PR description: branch, base, `merge-base`, `log --oneline base..HEAD`, `diff --stat` | `at-recall pr [--base ]` — the branch, its commits and its diffstat in one read | +| `curl https://api.koios.rest/…`, a retyped JSON body, a key in the command line | `at-cardano []` · `at-cardano query ` · `at-cardano endpoints` · `at-cardano run --scenario ` for several in order | | anything not listed | `at-describe` — the catalogue, one screen | - The exact lines · every mention · how many · which files → `at-peek slice :40-60` · @@ -424,6 +426,11 @@ nix develop -c cargo clippy --fix --allow-dirty --all-targets --all-features -- - `which at-peek` empty → `direnv exec . at-peek ` (and the same for `at-recall`, `at-crate` and `at-describe`). An agent's spawned shell does not inherit the devshell environment; an interactive one does. +- **Never wrap an `at-*` call in `nix develop`.** It works, which is what makes it the expensive + mistake: the nix evaluation is paid before the read starts, and the read itself is milliseconds. + Measured 2026-10-26: `at-peek stat` bare, 5 ms; `nix develop path:. --command at-peek stat`, + 7.5 s. If the binary is not on `PATH`, the answers are `direnv exec .` (~140 ms, reads the + realised `.direnv/`, needs no daemon) or installing it — never a devshell wrapper. - `at-crate` resolves a crate through the nearest `Cargo.lock`, not through memory or "latest". A crate named twice in the lock, or a file present in more than one build-script `out`, is an error naming the flag that chooses — never a guess. `--version` and `--build-dir` are those flags. @@ -439,7 +446,10 @@ nix develop -c cargo clippy --fix --allow-dirty --all-targets --all-features -- before editing it. - `at-peek` and `at-crate` run nothing at all; `at-recall` runs `git` and nothing else. That difference is why they are separate binaries, and why they are separate approvals if I have - tiered them. + tiered them. `at-cardano` makes HTTPS requests to Koios and nothing else: the host is fixed by + `--network`, the key is read from `~/.config/koios/api_key` (never printed), and `/submittx` is + refused. Never curl a Koios endpoint to avoid it; the curl is the generic executor this tool + replaces. - A **recipe** is one verb that composes the reads its sibling verbs use, so a count cannot disagree with the listing beside it: `at-recall pr` answers "write me a PR description" with the base, the branch's commits and the diffstat by kind, and a caveat naming tracked files that have changed @@ -451,7 +461,14 @@ nix develop -c cargo clippy --fix --allow-dirty --all-targets --all-features -- - `at-recall` answers `state`, `log`, `diff` and the `pr` recipe. `blame`, `show` and `churn`, and the `review` and `release` recipes, are designed and not written — ask for the one you want rather than reaching for `git`, and name the question, because that is what turns it into a verb. `at-crate` - answers `where`, `installed`, `files`, `slice`, `search` and `out`. + answers `where`, `installed`, `files`, `slice`, `search` and `out`. `at-cardano` is noun + aspect — + `at-cardano pool delegators `, `at-cardano tx utxos `, + `at-cardano account rewards ` — and `at-cardano --help` lists the nouns while + `at-cardano help ` lists that noun's aspects. Every read endpoint in the published Koios + spec has a verb; `at-cardano endpoints` proves it, and `query` is the long tail rather than the + way in. A question that takes two answers — the tip's height into `block`, a block's first + transaction into `tx` — is `at-cardano run --scenario `, where a `save` binds one value + out of an answer and the next step uses it; `at-cardano run --help` names the five actions. ## Toggle local [patch] blocks as a unit @@ -495,6 +512,47 @@ nix develop -c cargo tree --target wasm32-unknown-unknown -p tree, not the direct dependency. +## Never drive a terminal UI with tmux — use tui-shot or the crate's harness + + +**Never drive a TUI with tmux.** `tmux` is a generic executor: approving it approves `send-keys` +into any pane and `attach` to sessions it did not start, so it cannot be granted per-prefix. + +| Want | Use | +| --- | --- | +| a live screen, keys, a resize, a narrow terminal | `tui-shot` — one PTY, one workspace binary target, no shell | +| a frame while iterating on the renderer | the crate's own `TestBackend` harness | +| the second check | `tui-shot`'s `.json` · the harness's `run.json` | + +```sh +tui-shot run --scenario .tmp/factotum.dsl +tui-shot check --scenario .tmp/factotum.dsl # parse only, launch nothing +./target/release/ --state fixtures/x.json --plan examples/x.dsl +``` + +- One plan file drives both: `tui-shot` through the PTY, the harness against a captured view. The + grammar is `tools/tui-shot/SCENARIO.md` in `agent-playbook`. +- A harness-only field (`at=`, `status=`) is refused by `tui-shot` by name — omit it in a plan + that is meant to run under both. +- Wait for content, never sleep: `wait … contains=`, `absent=`, `idle=`, `exit=true`. +- Not: a hand-started tmux session, `send-keys`, `capture-pane`, or a fixed `sleep` between a key + and a screen read. + +### Why + +The factotum agent thread drove `factotum-tui` with `tmux new-session`, `send-keys`, a sleep, then +`capture-pane` for every state. The loop's speed was the sleeps, and every approval covered an +executor that can type into any pane on the machine. + +Measured on 2026-10-26 against the same TUI: a `tui-shot` run is 650–1100 ms depending on how many +waits it has; five captures through the crate's `TestBackend` harness are 2 ms in total, at +180–280 µs a frame. The harness needs no daemon, so a renderer can be reviewed before the service +it talks to exists. + +`tui-shot` refuses a path and a shell command by construction; the harness refuses to write +outside the repository. That is why either can be approved where tmux cannot. + + ## Bulk uploads to R2 run on the arm box, not the workstation diff --git a/crates/runtime/src/observe.rs b/crates/runtime/src/observe.rs index 0e10614..28f7497 100644 --- a/crates/runtime/src/observe.rs +++ b/crates/runtime/src/observe.rs @@ -119,28 +119,35 @@ struct SystemStatus { status: String, } -/// Read the runtime's world in four calls, gated on `system status`. +/// Read the runtime's world in five calls; `system status` is advisory. pub fn observe(program: &str) -> Result { - let status: SystemStatus = parse( - "`container system status`", - &run(program, &["system", "status", "--format", "json"])?, - )?; - if status.status != "running" { - return Err(RuntimeError::message(format!( - "the container runtime is not running (status: `{}`); run `container system start`", - status.status - ))); - } + // `system status` answers `unregistered` to a launchd session other than + // the one that started the runtime (an SSH login, typically) while `list` + // works from anywhere — so it must not gate. Ask it only to keep the + // actionable message for a genuinely stopped runtime, and let `list` decide. + let status = run(program, &["system", "status", "--format", "json"]) + .ok() + .and_then(|output| parse::("`container system status`", &output).ok()) + .map(|status| status.status); + + let containers = match run(program, &["list", "--all", "--format", "json"]) { + Ok(output) => parse("`container list`", &output)?, + Err(error) => { + if let Some(status) = status.filter(|status| status != "running") { + return Err(RuntimeError::message(format!( + "the container runtime is not running (status: `{status}`); run `container system start`" + ))); + } + return Err(error); + } + }; Ok(Observed { dns_domains: parse( "`container system dns list`", &run(program, &["system", "dns", "list", "--format", "json"])?, )?, - containers: parse( - "`container list`", - &run(program, &["list", "--all", "--format", "json"])?, - )?, + containers, volumes: parse( "`container volume list`", &run(program, &["volume", "list", "--format", "json"])?, diff --git a/flake.lock b/flake.lock index eba07bf..08faf1c 100644 --- a/flake.lock +++ b/flake.lock @@ -12,11 +12,11 @@ ] }, "locked": { - "lastModified": 1792917612, - "narHash": "sha256-hANbI3cvf+bu4h/k8A5M8Xxi5CNS+mnFr0sAe56tR2Y=", + "lastModified": 1793416464, + "narHash": "sha256-huRU0ZUb7386KT3nTZZfmvfSPVCskEbgKgoi0EDLfrQ=", "owner": "defrag-au", "repo": "agent-playbook", - "rev": "8eb99e1b1e854d6aa34c63a052966f854049cb32", + "rev": "e5a6120f8aa9e2e7a4226a014d617f84dd60e9ab", "type": "github" }, "original": { @@ -99,11 +99,11 @@ "nixpkgs": "nixpkgs_18" }, "locked": { - "lastModified": 1792925781, - "narHash": "sha256-yVIHTOWaqM7njyYMyH+ezKlTVaZUP+jEmt+2wd1TIV4=", + "lastModified": 1793417564, + "narHash": "sha256-Z99MHgBs/kGdy8Opyjgw+hTM4OoMNhiP7/BYvNP5XCo=", "owner": "defrag-au", "repo": "defrag-nix", - "rev": "be7ddfb3a0015fd5667f57e1cb1c678e729cd33a", + "rev": "6165c0881fbed79fd2c68a4ff67aa64a06498d69", "type": "github" }, "original": {