From f5e9446782751b69a0f2b09a580d1b8828d0580a Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Sun, 20 Sep 2026 00:06:07 -0700 Subject: [PATCH 1/3] fix(os): give the guest journal a budget it can satisfy SystemKeepFree=1G asked for more free space than a 2 GB CVM's /var/log has, and journald folds keep-free into the limit, so the declared 128M budget collapsed to the 16M floor. The System* settings were inert anyway: nothing in the image creates /var/log/journal, so journald has always used /run/log/journal, where RuntimeMaxUse=0 left it with no budget at all. --- os/common/rootfs/journald.conf | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/os/common/rootfs/journald.conf b/os/common/rootfs/journald.conf index 90a182d4d..8db4b50eb 100644 --- a/os/common/rootfs/journald.conf +++ b/os/common/rootfs/journald.conf @@ -1,7 +1,30 @@ [Journal] -SystemMaxUse=128M -SystemKeepFree=1G -SystemMaxFileSize=10M -SystemMaxFiles=10 -RuntimeMaxUse=0 +# The guest has no persistent journal, so say so and budget the one it has. +# +# Storage=auto means persistent only if /var/log/journal already exists, and +# nothing creates it: Debian's systemd package ships no /var/log entries +# (verified against systemd 257.13-1~deb13u1 from the pinned snapshot), its +# tmpfiles rule for that directory is `z` -- adjust if present, never create -- +# and no dstack recipe makes one. So journald has always written to +# /run/log/journal, where every System* setting below used to be inert and +# RuntimeMaxUse=0 was the only live one. journald reported the result as +# "Runtime Journal (...) is 8.0M, max 0B, 0B free": no budget at all. +# +# The System* settings would not have worked either. /var/log is an overlay +# whose upper layer is the /var/volatile tmpfs, so SystemKeepFree=1G asked for +# more free space than the whole filesystem has on a 2 GB CVM. journald folds +# keep-free into the limit -- MIN(MAX(used + free - keep_free, min_use), +# max_use) -- so the 128M budget collapsed to the 16M floor: "System Journal +# (...) is 8.0M, max 16.0M" on a 1 GiB /var/log, against "max 128.0M" on 8 GiB. +# +# Keeping the journal in /run rather than creating /var/log/journal is +# deliberate. Both are tmpfs and neither survives a reboot, but /var/volatile +# is also where /var/lib/docker's writable layer lives, and the images are what +# fills a small CVM. Every limit here is a size journald can satisfy in /run, +# and RuntimeKeepFree keeps its default 15% of the filesystem, which scales +# with the CVM instead of being a fixed floor larger than it. +Storage=volatile +RuntimeMaxUse=128M +RuntimeMaxFileSize=10M +RuntimeMaxFiles=10 ReadKMsg=no From 1666f89f176272f786b9f81030c9dd833d31bd92 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 22 Sep 2026 19:50:05 -0700 Subject: [PATCH 2/3] fix(os): let the runtime journal budget scale with guest memory --- os/common/rootfs/journald.conf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/os/common/rootfs/journald.conf b/os/common/rootfs/journald.conf index 8db4b50eb..37ce1c82a 100644 --- a/os/common/rootfs/journald.conf +++ b/os/common/rootfs/journald.conf @@ -6,7 +6,7 @@ # (verified against systemd 257.13-1~deb13u1 from the pinned snapshot), its # tmpfiles rule for that directory is `z` -- adjust if present, never create -- # and no dstack recipe makes one. So journald has always written to -# /run/log/journal, where every System* setting below used to be inert and +# /run/log/journal, where every System* setting used to be inert and # RuntimeMaxUse=0 was the only live one. journald reported the result as # "Runtime Journal (...) is 8.0M, max 0B, 0B free": no budget at all. # @@ -14,17 +14,17 @@ # whose upper layer is the /var/volatile tmpfs, so SystemKeepFree=1G asked for # more free space than the whole filesystem has on a 2 GB CVM. journald folds # keep-free into the limit -- MIN(MAX(used + free - keep_free, min_use), -# max_use) -- so the 128M budget collapsed to the 16M floor: "System Journal -# (...) is 8.0M, max 16.0M" on a 1 GiB /var/log, against "max 128.0M" on 8 GiB. +# max_use) -- so the 128M budget collapsed to the 16M floor. # # Keeping the journal in /run rather than creating /var/log/journal is # deliberate. Both are tmpfs and neither survives a reboot, but /var/volatile # is also where /var/lib/docker's writable layer lives, and the images are what -# fills a small CVM. Every limit here is a size journald can satisfy in /run, -# and RuntimeKeepFree keeps its default 15% of the filesystem, which scales -# with the CVM instead of being a fixed floor larger than it. +# fills a small CVM. +# +# RuntimeMaxUse and RuntimeKeepFree stay at journald's defaults (10% and 15% +# of /run, itself 20% of RAM), so the budget scales with guest memory: a fixed +# 128M would take most of /run on a 512M CVM. The file limits cap it at 100M. Storage=volatile -RuntimeMaxUse=128M RuntimeMaxFileSize=10M RuntimeMaxFiles=10 ReadKMsg=no From 4558315539d250a2bbebd1eab07260381ff7b137 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 22 Sep 2026 19:50:49 -0700 Subject: [PATCH 3/3] fix(os): trim journald.conf comment --- os/common/rootfs/journald.conf | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/os/common/rootfs/journald.conf b/os/common/rootfs/journald.conf index 37ce1c82a..763b6e1f3 100644 --- a/os/common/rootfs/journald.conf +++ b/os/common/rootfs/journald.conf @@ -1,29 +1,8 @@ [Journal] -# The guest has no persistent journal, so say so and budget the one it has. -# -# Storage=auto means persistent only if /var/log/journal already exists, and -# nothing creates it: Debian's systemd package ships no /var/log entries -# (verified against systemd 257.13-1~deb13u1 from the pinned snapshot), its -# tmpfiles rule for that directory is `z` -- adjust if present, never create -- -# and no dstack recipe makes one. So journald has always written to -# /run/log/journal, where every System* setting used to be inert and -# RuntimeMaxUse=0 was the only live one. journald reported the result as -# "Runtime Journal (...) is 8.0M, max 0B, 0B free": no budget at all. -# -# The System* settings would not have worked either. /var/log is an overlay -# whose upper layer is the /var/volatile tmpfs, so SystemKeepFree=1G asked for -# more free space than the whole filesystem has on a 2 GB CVM. journald folds -# keep-free into the limit -- MIN(MAX(used + free - keep_free, min_use), -# max_use) -- so the 128M budget collapsed to the 16M floor. -# -# Keeping the journal in /run rather than creating /var/log/journal is -# deliberate. Both are tmpfs and neither survives a reboot, but /var/volatile -# is also where /var/lib/docker's writable layer lives, and the images are what -# fills a small CVM. -# -# RuntimeMaxUse and RuntimeKeepFree stay at journald's defaults (10% and 15% -# of /run, itself 20% of RAM), so the budget scales with guest memory: a fixed -# 128M would take most of /run on a 512M CVM. The file limits cap it at 100M. +# Nothing creates /var/log/journal, so the journal lives in /run. Keep it there: +# /var/volatile is shared with docker's writable layer. +# RuntimeMaxUse/RuntimeKeepFree stay at their defaults (10%/15% of /run) so the +# budget scales with guest memory. Storage=volatile RuntimeMaxFileSize=10M RuntimeMaxFiles=10