From c518ab6073e7717c0ade53dea223733f1154cd24 Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Wed, 15 Jul 2026 01:53:23 +0530 Subject: [PATCH 1/2] vsockexec: add opt-in reconnect mode (-r) for resilient forwarding By default vsockexec dials the host once, hands the socket to the child as its stdio, and execs into it. If the host listener is absent when the guest starts, or the connection later drops, the child's writes fail and it exits. Because the child runs under PID 1 in the UVM, its death makes PID 1 exit and the guest kernel panics, taking the VM down. Add a new -r flag that keeps vsockexec running as a relay between the child and the host instead of exec'ing into it: - the child writes to a pipe (which never breaks), not to the socket; - vsockexec keeps the host connection itself and forwards the pipe to it; - it waits and retries if the host is not listening yet (late connect); - it re-dials and resends if the connection drops (reconnect); - the child's exit status is propagated, so PID 1 still sees the real exit code and shutdown semantics are unchanged. Without -r the behavior is byte-for-byte identical to before, so no existing caller is affected. stdout and stderr sharing one port continue to share a single host connection. Signed-off-by: Harsh Rawat --- vsockexec/vsockexec.c | 179 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 177 insertions(+), 2 deletions(-) diff --git a/vsockexec/vsockexec.c b/vsockexec/vsockexec.c index fa71b2e6ea..0677a97e20 100644 --- a/vsockexec/vsockexec.c +++ b/vsockexec/vsockexec.c @@ -1,13 +1,21 @@ // vsockexec opens vsock connections for the specified stdio descriptors and // then execs the specified process. +// +// With -r ("reconnect mode") it does NOT hand the connection to the process. +// Instead it keeps running as a middleman (a "relay") between the process and +// the host, so that a host connection which is missing at startup or which +// drops later never reaches the process. See run_relay below for the details. #include "vsock.h" #include #include +#include #include #include #include #include +#include +#include #include #ifdef USE_TCP @@ -33,16 +41,171 @@ static int opentcp(unsigned short port) { return s; } +// sleep_ms pauses for the given number of milliseconds. We use it to wait a +// short moment between connection attempts instead of retrying back-to-back and +// burning CPU. +static void sleep_ms(long ms) { + struct timespec ts = {ms / 1000, (ms % 1000) * 1000000L}; + nanosleep(&ts, NULL); +} + +// dial_retry opens a connection to the host on the given port and never gives +// up. If the host side isn't listening yet (it may come up a little later) or +// has gone away (for example the VM was moved to a different host), it just +// keeps trying until it succeeds. This is what lets the guest wait for the host +// instead of failing, which is important because failing here would take the +// whole VM down. +static int dial_retry(unsigned int port) { + for (;;) { + // Try to connect once. Normal builds use vsock to reach the host; the + // tcp branch only exists for local testing. + int s = tcpmode ? opentcp(port) : openvsock(VMADDR_CID_HOST, port); + if (s >= 0) { + return s; // success: we now have a live connection to the host + } + // Couldn't connect yet. Wait a moment and then try again. + sleep_ms(100); + } +} + +// run_relay implements "reconnect mode" (-r). +// +// Normally vsockexec connects a socket to the host and hands that socket +// straight to the program as its stdout/stderr. The problem: if that host +// connection breaks, the program's writes fail and the program dies. Since the +// program is a child of PID 1 inside the VM, a dying program makes PID 1 exit, +// which makes the Linux kernel panic and the VM crash. +// +// run_relay avoids that by sitting in the MIDDLE and never letting the program +// touch the host connection directly: +// +// program --writes--> [ pipe ] --run_relay reads--> [ host connection ] +// +// The program only ever writes into a pipe, which never breaks. run_relay reads +// from that pipe and forwards the data to the host. If the host connection is +// missing or drops, only run_relay notices; it quietly reconnects and keeps +// going, so the program never sees a problem and stays alive. +static int run_relay(unsigned int ports[3], char** child_argv) { + // In reconnect mode we forward the program's OUTPUT (stdout and/or stderr) + // to one host port. Use whichever output port was requested. + unsigned int port = ports[2] != 0 ? ports[2] : ports[1]; + if (port == 0) { + // Nothing to forward: reconnect mode is only meaningful with -o or -e. + fprintf(stderr, "vsockexec: -r requires -o or -e\n"); + return 1; + } + + // Writing to a connection whose other end has gone away normally kills this + // process with a SIGPIPE signal. Tell the system to ignore that signal, so + // a dropped connection instead shows up as an ordinary write error that we + // can catch below and recover from by reconnecting. + signal(SIGPIPE, SIG_IGN); + + // Create a pipe: a simple one-way channel. Anything written to p[1] (the + // write end) can be read back from p[0] (the read end). The program will + // write into p[1]; we (the relay) will read from p[0]. + int p[2]; + if (pipe(p) < 0) { + perror("pipe"); + return 1; + } + + // Split into two processes. The child will turn into the program; the + // parent stays behind and becomes the relay/middleman. + pid_t pid = fork(); + if (pid < 0) { + perror("fork"); + return 1; + } + if (pid == 0) { + // ---------------- Child: becomes the program (e.g. GCS) ------------- + signal(SIGPIPE, SIG_DFL); // restore normal signal behavior for the program + close(p[0]); // the program only writes, so close the read end + // Redirect the program's stdout/stderr into the pipe instead of a real + // socket. After this, everything the program prints goes into the pipe. + if (ports[1] != 0) { + dup2(p[1], 1); // stdout -> pipe + } + if (ports[2] != 0) { + dup2(p[1], 2); // stderr -> pipe + } + close(p[1]); // the fds above are now our copies; close the original + // Replace this child process with the requested program and run it. + // CodeQL [SM01925] designed to forward stdio over VSOCK and then exec the command-line arguments (always ./cmd/gcs) + execvp(child_argv[0], child_argv); + // We only reach here if the program failed to start. + fprintf(stderr, "execvp: %s: %s\n", child_argv[0], strerror(errno)); + _exit(127); + } + + // -------------------- Parent: the relay/middleman ---------------------- + close(p[1]); // the relay only reads, so close the write end + + // Open the connection to the host, waiting for the host if it isn't ready. + int sock = dial_retry(port); + + // Main loop: keep copying whatever the program writes into the pipe out to + // the host connection, for as long as the program is running. + char buf[65536]; + for (;;) { + // Read the next chunk of the program's output from the pipe. + ssize_t n = read(p[0], buf, sizeof(buf)); + if (n < 0) { + if (errno == EINTR) { + continue; // a signal interrupted the read; simply try again + } + break; // unexpected error reading the pipe; stop + } + if (n == 0) { + break; // the pipe reached end-of-file: the program has exited + } + // Send those n bytes to the host. Keep sending until every byte is out, + // reconnecting if the connection breaks in the middle of sending. + for (ssize_t off = 0; off < n;) { + ssize_t w = write(sock, buf + off, (size_t)(n - off)); + if (w < 0) { + if (errno == EINTR) { + continue; // interrupted by a signal; retry the write + } + // The connection dropped (for example the host went away). Drop + // the dead socket, get a fresh connection, and resend the bytes + // that had not been sent yet (off marks how far we got). + close(sock); + sock = dial_retry(port); + continue; + } + off += w; // w bytes were sent; move past them and continue + } + } + close(sock); + + // The program has finished. Wait for it and then exit with the SAME status + // it did, so that whoever launched us (PID 1) sees the program's real exit + // code and behaves exactly as if it had run the program directly. + int status = 0; + while (waitpid(pid, &status, 0) < 0 && errno == EINTR) { + // a signal interrupted the wait; keep waiting + } + if (WIFEXITED(status)) { + return WEXITSTATUS(status); // program exited normally: pass on its code + } + if (WIFSIGNALED(status)) { + return 128 + WTERMSIG(status); // program was killed by a signal + } + return 1; +} + _Noreturn static void usage(const char* argv0) { - fprintf(stderr, "%s [-i port] [-o port] [-e port] -- program [args...]\n", argv0); + fprintf(stderr, "%s [-r] [-i port] [-o port] [-e port] -- program [args...]\n", argv0); exit(1); } int main(int argc, char** argv) { unsigned int ports[3] = {0}; int sockets[3] = {-1, -1, -1}; + int reconnect = 0; int c; - while ((c = getopt(argc, argv, "+i:o:e:")) != -1) { + while ((c = getopt(argc, argv, "+i:o:e:r")) != -1) { switch (c) { case 'i': ports[0] = strtoul(optarg, NULL, 10); @@ -56,6 +219,11 @@ int main(int argc, char** argv) { ports[2] = strtoul(optarg, NULL, 10); break; + case 'r': + // -r turns on reconnect mode (resilient forwarding via run_relay). + reconnect = 1; + break; + default: usage(argv[0]); } @@ -66,6 +234,13 @@ int main(int argc, char** argv) { usage(argv[0]); } + if (reconnect) { + // Resilient path: wait for the host if it isn't listening yet and + // reconnect if the connection later drops. The plain connect-and-exec + // path below (used when -r is absent) does neither. + return run_relay(ports, argv + optind); + } + for (int i = 0; i < 3; i++) { if (ports[i] != 0) { int j; From 96c75005179b0bb625b9b4492bd955e6a84874c3 Mon Sep 17 00:00:00 2001 From: Harsh Rawat Date: Wed, 15 Jul 2026 02:41:29 +0530 Subject: [PATCH 2/2] keep host-side GCS logs across live migration via vsockexec -r Wire the reconnect mode (-r) added in the previous commit into the LCOW live-migration path so live-migratable pods keep host-side GCS logging instead of forgoing it. Before, these pods dropped GCS log forwarding entirely: a plain vsockexec would block on its outbound connect and stall guest init when the host log listener was absent, so the wrapper and listener were skipped. Reconnect mode removes that constraint: - buildGCSCommand emits /bin/vsockexec -r for live-migratable sandboxes, so the guest tolerates a missing listener and re-dials LinuxLogVsockPort after the connection drops; - StartWithMigrationOptions arms a fresh listener on the destination before start, so the resumed guest can reconnect and resume streaming; - setupLoggingListener no longer early-returns and Resume no longer force-closes logOutputDone. Signed-off-by: Harsh Rawat --- internal/builder/vm/lcow/kernel_args.go | 39 +++++------ internal/builder/vm/lcow/sandbox_options.go | 4 -- internal/builder/vm/lcow/specs.go | 10 ++- internal/builder/vm/lcow/specs_test.go | 74 +++++++++------------ internal/controller/vm/save_lcow.go | 34 ++++++---- internal/controller/vm/vm_lcow.go | 35 ++-------- internal/controller/vm/vm_migration.go | 36 ++++++++++ 7 files changed, 120 insertions(+), 112 deletions(-) diff --git a/internal/builder/vm/lcow/kernel_args.go b/internal/builder/vm/lcow/kernel_args.go index d5822b73b8..d828226aba 100644 --- a/internal/builder/vm/lcow/kernel_args.go +++ b/internal/builder/vm/lcow/kernel_args.go @@ -27,7 +27,6 @@ func buildKernelArgs( kernelDirect bool, hasConsole bool, rootFsFile string, - liveMigrationSupportEnabled bool, ) (string, error) { log.G(ctx).WithField("rootFsFile", rootFsFile).Debug("buildKernelArgs: starting kernel arguments construction") @@ -83,7 +82,7 @@ func buildKernelArgs( // 8. Init arguments (passed after "--" separator) initArgs := buildInitArgs(ctx, opts, annotations, - writableOverlayDirs, disableTimeSyncService, processDumpLocation, rootFsFile, hasConsole, liveMigrationSupportEnabled) + writableOverlayDirs, disableTimeSyncService, processDumpLocation, rootFsFile, hasConsole) args = append(args, "--", initArgs) result := strings.Join(args, " ") @@ -153,7 +152,6 @@ func buildInitArgs( processDumpLocation string, rootFsFile string, hasConsole bool, - liveMigrationSupportEnabled bool, ) string { log.G(ctx).WithFields(logrus.Fields{ "rootFsFile": rootFsFile, @@ -163,7 +161,7 @@ func buildInitArgs( entropyArgs := fmt.Sprintf("-e %d", vmutils.LinuxEntropyVsockPort) // Build GCS execution command - gcsCmd := buildGCSCommand(opts, annotations, disableTimeSyncService, processDumpLocation, liveMigrationSupportEnabled) + gcsCmd := buildGCSCommand(ctx, opts, annotations, disableTimeSyncService, processDumpLocation) // Construct init arguments var initArgsList []string @@ -194,12 +192,26 @@ func buildInitArgs( // buildGCSCommand constructs the GCS (Guest Compute Service) command line. func buildGCSCommand( + ctx context.Context, opts *runhcsoptions.Options, annotations map[string]string, disableTimeSyncService bool, processDumpLocation string, - liveMigrationSupportEnabled bool, ) string { + // Start with vsockexec wrapper + var cmdParts []string + cmdParts = append(cmdParts, "/bin/vsockexec") + + // When live migration is enabled, run vsockexec in reconnect mode (-r) so + // guest logging tolerates the host log listener being absent at boot and + // reconnects to the destination host's listener after a migration. + if oci.ParseAnnotationsBool(ctx, annotations, shimannotations.LiveMigrationSupportEnabled, false) { + cmdParts = append(cmdParts, "-r") + } + + // Add logging vsock port + cmdParts = append(cmdParts, fmt.Sprintf("-e %d", vmutils.LinuxLogVsockPort)) + // Determine log level logLevel := "info" if opts != nil && opts.LogLevel != "" { @@ -232,19 +244,8 @@ func buildGCSCommand( gcsParts = append(gcsParts, s) } - gcsCmd := strings.Join(gcsParts, " ") - - // Live-migratable pods skip the /bin/vsockexec wrapper. The wrapper exists - // solely to forward GCS stderr to the host-side log listener, but that listener - // is host-local state that live migration does not transfer, so the host - // does not run it for these pods. - // Without a listener, vsockexec's outbound connect would block and stall guest init, - // so we emit /bin/gcs directly instead. - if liveMigrationSupportEnabled { - return gcsCmd - } + // Combine vsockexec and GCS command + cmdParts = append(cmdParts, strings.Join(gcsParts, " ")) - // vsockexec `-e ` wires gcs's stderr to LinuxLogVsockPort, which - // the host listener reads and republishes. - return fmt.Sprintf("/bin/vsockexec -e %d %s", vmutils.LinuxLogVsockPort, gcsCmd) + return strings.Join(cmdParts, " ") } diff --git a/internal/builder/vm/lcow/sandbox_options.go b/internal/builder/vm/lcow/sandbox_options.go index 37c978a170..492d3678b5 100644 --- a/internal/builder/vm/lcow/sandbox_options.go +++ b/internal/builder/vm/lcow/sandbox_options.go @@ -25,10 +25,6 @@ type SandboxOptions struct { // ConfidentialConfig carries confidential computing fields that are not // part of the HCS document but are needed for confidential VM setup. ConfidentialConfig *ConfidentialConfig - - // LiveMigrationSupportEnabled indicates that the live migration feature set is - // enabled for the sandbox, constraining it to migration-compatible features. - LiveMigrationSupportEnabled bool } // ConfidentialConfig carries confidential computing configuration that is not diff --git a/internal/builder/vm/lcow/specs.go b/internal/builder/vm/lcow/specs.go index 18074e3bc4..2229e0da63 100644 --- a/internal/builder/vm/lcow/specs.go +++ b/internal/builder/vm/lcow/specs.go @@ -222,7 +222,6 @@ func BuildSandboxConfig( bootOptions.LinuxKernelDirect != nil, // isKernelDirectBoot comPorts != nil, // hasConsole filepath.Base(rootFsFullPath), - sandboxOptions.LiveMigrationSupportEnabled, ) if err != nil { return nil, nil, fmt.Errorf("failed to build kernel args: %w", err) @@ -331,11 +330,10 @@ func parseSandboxOptions(ctx context.Context, platform string, annotations map[s log.G(ctx).WithField("platform", platform).Debug("parseSandboxOptions: starting sandbox options parsing") sandboxOptions := &SandboxOptions{ // Extract architecture from platform string (e.g., "linux/amd64" -> "amd64") - Architecture: platform[strings.IndexByte(platform, '/')+1:], - FullyPhysicallyBacked: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.FullyPhysicallyBacked, false), - PolicyBasedRouting: oci.ParseAnnotationsBool(ctx, annotations, iannotations.NetworkingPolicyBasedRouting, false), - NoWritableFileShares: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DisableWritableFileShares, false), - LiveMigrationSupportEnabled: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.LiveMigrationSupportEnabled, false), + Architecture: platform[strings.IndexByte(platform, '/')+1:], + FullyPhysicallyBacked: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.FullyPhysicallyBacked, false), + PolicyBasedRouting: oci.ParseAnnotationsBool(ctx, annotations, iannotations.NetworkingPolicyBasedRouting, false), + NoWritableFileShares: oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DisableWritableFileShares, false), } // Determine if this is a confidential VM early, as it affects boot options parsing diff --git a/internal/builder/vm/lcow/specs_test.go b/internal/builder/vm/lcow/specs_test.go index ac7d0b5842..6b04df1ff2 100644 --- a/internal/builder/vm/lcow/specs_test.go +++ b/internal/builder/vm/lcow/specs_test.go @@ -2129,34 +2129,35 @@ func TestBuildSandboxConfig_CPUClamping(t *testing.T) { } // TestBuildSandboxConfig_LiveMigration validates the wiring for the -// io.microsoft.migration.support-enabled sandbox annotation. The annotation is parsed -// into SandboxOptions.LiveMigrationSupportEnabled and threaded down into the kernel -// command line: live-migratable sandboxes must skip the /bin/vsockexec wrapper -// (which would otherwise stall init waiting for a host log listener that the -// LM-enabled host does not run), while non-LM sandboxes must continue to use -// vsockexec so that GCS stderr is forwarded over LinuxLogVsockPort. +// io.microsoft.migration.support-enabled sandbox annotation. The annotation is +// parsed directly when building the GCS command: live-migratable sandboxes run +// the /bin/vsockexec wrapper in reconnect mode (-r) so guest logging tolerates a +// missing host log listener and reconnects after a migration, while non-LM +// sandboxes use the plain wrapper to forward GCS stderr over LinuxLogVsockPort. func TestBuildSandboxConfig_LiveMigration(t *testing.T) { ctx := context.Background() validBootFilesPath := newBootFilesPath(t) defaultOpts := defaultSandboxOpts(validBootFilesPath) - // Pre-format the vsockexec prefix once so the assertions are obviously - // driven by the same constant the production code uses. + // Pre-format the vsockexec prefixes once so the assertions are obviously + // driven by the same constants the production code uses. vsockexecPrefix := fmt.Sprintf("/bin/vsockexec -e %d", vmutils.LinuxLogVsockPort) + // When live migration is enabled the wrapper runs in reconnect mode (-r). + vsockexecReconnectPrefix := fmt.Sprintf("/bin/vsockexec -r -e %d", vmutils.LinuxLogVsockPort) tests := []specTestCase{ { name: "live migration disabled by default", validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=false by default, got true") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, vsockexecPrefix) { t.Errorf("expected vsockexec wrapper %q in kernel args (LM disabled), got %q", vsockexecPrefix, kernelArgs) } + if strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected no vsockexec reconnect mode when LM disabled, got %q", kernelArgs) + } if !strings.Contains(kernelArgs, "/bin/gcs") { t.Errorf("expected /bin/gcs in kernel args, got %q", kernelArgs) } @@ -2171,17 +2172,17 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=false when annotation=\"false\", got true") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, vsockexecPrefix) { t.Errorf("expected vsockexec wrapper %q in kernel args, got %q", vsockexecPrefix, kernelArgs) } + if strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected no vsockexec reconnect mode when LM disabled, got %q", kernelArgs) + } }, }, { - name: "live migration enabled drops vsockexec wrapper", + name: "live migration enabled uses vsockexec reconnect mode", spec: &vm.Spec{ Annotations: map[string]string{ shimannotations.LiveMigrationSupportEnabled: "true", @@ -2189,21 +2190,14 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if !sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=true when annotation=\"true\", got false") - } kernelArgs := getKernelArgs(doc) - // The vsockexec wrapper must not appear at all when LM is on: - // neither the prefix nor the binary path on its own. - if strings.Contains(kernelArgs, "vsockexec") { - t.Errorf("expected no vsockexec in kernel args when LM enabled, got %q", kernelArgs) + // When LM is on the wrapper runs in reconnect mode (-r) so the + // guest tolerates a missing or dropped host log listener. + if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected vsockexec reconnect wrapper %q in kernel args when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs) } - if strings.Contains(kernelArgs, fmt.Sprintf("-e %d", vmutils.LinuxLogVsockPort)) { - t.Errorf("expected no log vsock port (%d) wiring when LM enabled, got %q", vmutils.LinuxLogVsockPort, kernelArgs) - } - // /bin/gcs must still be invoked - just without the wrapper. if !strings.Contains(kernelArgs, "/bin/gcs") { - t.Errorf("expected /bin/gcs in kernel args even when LM enabled, got %q", kernelArgs) + t.Errorf("expected /bin/gcs in kernel args when LM enabled, got %q", kernelArgs) } }, }, @@ -2221,22 +2215,19 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if !sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=true, got false") - } kernelArgs := getKernelArgs(doc) - // Other GCS flags must still be threaded through the command - // even when the vsockexec wrapper is removed. + // Other GCS flags must still be threaded through the reconnect + // wrapper when LM is enabled. if !strings.Contains(kernelArgs, "-loglevel debug") { t.Errorf("expected -loglevel debug in kernel args when LM enabled, got %q", kernelArgs) } - if strings.Contains(kernelArgs, "vsockexec") { - t.Errorf("expected no vsockexec when LM enabled, got %q", kernelArgs) + if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected vsockexec reconnect wrapper %q when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs) } }, }, { - name: "live migration with disable time sync still drops vsockexec", + name: "live migration with disable time sync uses vsockexec reconnect mode", spec: &vm.Spec{ Annotations: map[string]string{ shimannotations.LiveMigrationSupportEnabled: "true", @@ -2245,15 +2236,12 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if !sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=true, got false") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, "-disable-time-sync") { t.Errorf("expected -disable-time-sync flag in kernel args, got %q", kernelArgs) } - if strings.Contains(kernelArgs, "vsockexec") { - t.Errorf("expected no vsockexec when LM enabled, got %q", kernelArgs) + if !strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected vsockexec reconnect wrapper %q when LM enabled, got %q", vsockexecReconnectPrefix, kernelArgs) } }, }, @@ -2269,13 +2257,13 @@ func TestBuildSandboxConfig_LiveMigration(t *testing.T) { }, validate: func(t *testing.T, doc *hcsschema.ComputeSystem, sandboxOpts *SandboxOptions) { t.Helper() - if sandboxOpts.LiveMigrationSupportEnabled { - t.Errorf("expected LiveMigrationSupportEnabled=false on invalid annotation value, got true") - } kernelArgs := getKernelArgs(doc) if !strings.Contains(kernelArgs, vsockexecPrefix) { t.Errorf("expected vsockexec wrapper %q in kernel args, got %q", vsockexecPrefix, kernelArgs) } + if strings.Contains(kernelArgs, vsockexecReconnectPrefix) { + t.Errorf("expected no vsockexec reconnect mode on invalid LM annotation, got %q", kernelArgs) + } }, }, } diff --git a/internal/controller/vm/save_lcow.go b/internal/controller/vm/save_lcow.go index 29f330c74a..df4f0ee1e5 100644 --- a/internal/controller/vm/save_lcow.go +++ b/internal/controller/vm/save_lcow.go @@ -18,6 +18,7 @@ import ( "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/go-winio" + "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) @@ -222,6 +223,27 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error { if err := c.guest.ResumeConnection(ctx); err != nil { return fmt.Errorf("resume guest connection: %w", err) } + + // The blackout also dropped the source's GCS log connection, which tore + // down its listener and closed logOutputDone. Install a fresh signal and + // re-arm the listener so the resumed guest's reconnect-mode vsockexec can + // reconnect and host-side logs resume. The accept runs in the background + // (WithoutCancel so it outlives this call; AcceptConnection still returns + // on VM exit) so a slow guest re-dial cannot stall resume. + c.logOutputDone = make(chan struct{}) + g, gctx := errgroup.WithContext(ctx) + defer func() { + _ = g.Wait() + }() + + if err := c.setupLoggingListener(gctx, g); err != nil { + return fmt.Errorf("re-arm logging listener on resume: %w", err) + } + + // Collect any errors from establishing the log connection. + if err := g.Wait(); err != nil { + return err + } default: // Destination: reuse the connection already armed at start. if err := c.guest.CreateConnection(ctx, false); err != nil { @@ -252,18 +274,6 @@ func (c *Controller) Resume(ctx context.Context, rebuildBridge bool) error { c.vmState = StateRunning - if c.sandboxOptions != nil { - c.sandboxOptions.LiveMigrationSupportEnabled = true - } - - // Destination never ran setupLoggingListener; close so [Controller.Wait] - // does not block. Already closed on source — receive falls through. - select { - case <-c.logOutputDone: - default: - close(c.logOutputDone) - } - log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("resumed VM from migration") return nil } diff --git a/internal/controller/vm/vm_lcow.go b/internal/controller/vm/vm_lcow.go index 3c548df04c..7c5aad253f 100644 --- a/internal/controller/vm/vm_lcow.go +++ b/internal/controller/vm/vm_lcow.go @@ -12,7 +12,6 @@ import ( "github.com/Microsoft/hcsshim/internal/controller/device/plan9" "github.com/Microsoft/hcsshim/internal/controller/network" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" - "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/internal/vm/vmmanager" "github.com/Microsoft/hcsshim/internal/vm/vmutils" @@ -174,30 +173,10 @@ func (c *Controller) setupEntropyListener(ctx context.Context, group *errgroup.G // running inside the Linux VM. The logs are parsed and // forwarded to the host's logging system for monitoring and debugging. func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.Group) error { - // Live-migratable sandboxes intentionally run without a host-side GCS log - // listener. - // - // The log listener is host-local state: GCS inside the guest connects out to - // a host-side hvsocket on LinuxLogVsockPort and streams its stderr to it. That - // connection, and the goroutine reading from it, are bound to the *source* - // host and are not part of the guest state that live migration transfers. - // After the VM is migrated to a destination host there is no equivalent - // listener to reconnect to, so a guest that depended on the log socket would - // block on its outbound connect and stall the boot path. To keep the guest - // migratable we skip the listener here and drop the matching /bin/vsockexec - // wrapper from the kernel command line, so GCS never attempts the connection. - // - // Re-enabling host-side log collection for live-migratable pods requires a - // migration-aware log transport: GCS must tolerate the listener going away - // and reconnect to a freshly established listener on the destination host once - // migration completes, and the host must (re)create the listener and re-attach - // the log-parsing goroutine on the destination. Until that work lands we forgo - // host-side GCS logs for these pods. - if c.sandboxOptions != nil && c.sandboxOptions.LiveMigrationSupportEnabled { - log.G(ctx).Info("skipping GCS log listener: pod is live-migratable") - close(c.logOutputDone) - return nil - } + // Capture the current signal channel: if the listener is re-armed later + // (e.g. source rollback), this goroutine must close the channel it was + // started with, never a freshly installed one. + done := c.logOutputDone // The GCS will connect to this port to stream log output. logConn, err := winio.ListenHvsock(&winio.HvsockAddr{ @@ -205,7 +184,7 @@ func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.G ServiceID: winio.VsockServiceID(vmutils.LinuxLogVsockPort), }) if err != nil { - close(c.logOutputDone) + close(done) return fmt.Errorf("failed to listen on hvSocket for logs: %w", err) } @@ -215,7 +194,7 @@ func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.G // Accept the connection from the GCS. conn, err := vmmanager.AcceptConnection(ctx, c.uvm, logConn, true) if err != nil { - close(c.logOutputDone) + close(done) return fmt.Errorf("failed to accept connection on hvSocket for logs: %w", err) } @@ -228,7 +207,7 @@ func (c *Controller) setupLoggingListener(ctx context.Context, group *errgroup.G // Signal that log output processing has completed. // This allows Wait() to ensure all logs are processed before returning. - close(c.logOutputDone) + close(done) }() return nil diff --git a/internal/controller/vm/vm_migration.go b/internal/controller/vm/vm_migration.go index 267669475a..6d3b8e86c4 100644 --- a/internal/controller/vm/vm_migration.go +++ b/internal/controller/vm/vm_migration.go @@ -6,14 +6,17 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/Microsoft/go-winio" + "golang.org/x/sync/errgroup" "github.com/Microsoft/hcsshim/internal/gcs/prot" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" hcs "github.com/Microsoft/hcsshim/internal/hcs/v2" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/logfields" + "github.com/Microsoft/hcsshim/internal/vm/vmutils" ) // compatibilityInfoProperty is the HCS property name used to retrieve the @@ -32,6 +35,20 @@ func (c *Controller) InitializeLiveMigrationOnSource(ctx context.Context, option return fmt.Errorf("cannot initialize live migration on source: VM is in state %s", c.vmState) } + // Live migration requires the guest log relay to run in reconnect mode so it can + // re-dial the destination host's log listener after the move. + var kernelCmdLine string + if chipset := c.hcsDocument.VirtualMachine.Chipset; chipset != nil { + if chipset.LinuxKernelDirect != nil { + kernelCmdLine = chipset.LinuxKernelDirect.KernelCmdLine + } else if chipset.Uefi != nil && chipset.Uefi.BootThis != nil { + kernelCmdLine = chipset.Uefi.BootThis.OptionalData + } + } + if !strings.Contains(kernelCmdLine, fmt.Sprintf("/bin/vsockexec -r -e %d", vmutils.LinuxLogVsockPort)) { + return fmt.Errorf("cannot initialize live migration on source: VM was not created with live-migration support enabled") + } + // Hand the initialize request to the HCS for the UVM. if err := c.uvm.InitializeLiveMigrationOnSource(ctx, options); err != nil { return fmt.Errorf("failed to initialize live migration on source: %w", err) @@ -115,6 +132,19 @@ func (c *Controller) StartWithMigrationOptions(ctx context.Context, config *hcs. return fmt.Errorf("cannot start with migration options: VM is in state %s", c.vmState) } + g, gctx := errgroup.WithContext(ctx) + defer func() { + _ = g.Wait() + }() + + // The source's log listener is host-local and does not transfer with the + // migration. Arm a fresh listener on the destination before start so the + // resumed guest's vsockexec (reconnect mode) can re-dial LinuxLogVsockPort + // and resume streaming GCS logs without racing the guest's dial. + if err := c.setupLoggingListener(gctx, g); err != nil { + return fmt.Errorf("failed to set up logging listener: %w", err) + } + // Arm the host-side GCS listener before start so the guest's dial cannot race it. if err := c.guest.PrepareConnection(winio.VsockServiceID(prot.LinuxGcsVsockPort)); err != nil { return fmt.Errorf("prepare destination gcs connection: %w", err) @@ -127,6 +157,12 @@ func (c *Controller) StartWithMigrationOptions(ctx context.Context, config *hcs. // Watch for VM exit in the background. go c.waitForVMExit(ctx) + + // Collect any errors from establishing the log connection. + if err := g.Wait(); err != nil { + return err + } + c.vmState = StateDestinationMigrationStarted log.G(ctx).WithField(logfields.UVMID, c.vmID).Debug("started destination VM with migration options")