diff --git a/Dockerfile b/Dockerfile index 826c9a6..840e4cd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ # - containerd shim for spinbox runtime # Base image versions -ARG GO_VERSION=1.25.5 +ARG GO_VERSION=1.26.2 ARG BASE_DEBIAN_DISTRO="bookworm" ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" diff --git a/Dockerfile.qemu b/Dockerfile.qemu index 468c0ef..c9e98f9 100644 --- a/Dockerfile.qemu +++ b/Dockerfile.qemu @@ -4,7 +4,7 @@ FROM ubuntu:25.10 AS builder # Build arguments -ARG QEMU_VERSION=10.2.0 +ARG QEMU_VERSION=10.2.2 ARG JOBS=8 ARG DEBIAN_FRONTEND=noninteractive diff --git a/api/next.pb.txt b/api/next.pb.txt index e41e992..e69de29 100755 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -1,503 +0,0 @@ -file { - name: "github.com/spin-stack/spinbox/api/services/bundle/v1/bundle.proto" - package: "containerd.vminitd.services.bundle.v1" - message_type { - name: "CreateRequest" - field { - name: "id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "id" - } - field { - name: "files" - number: 2 - label: LABEL_REPEATED - type: TYPE_MESSAGE - type_name: ".containerd.vminitd.services.bundle.v1.CreateRequest.FilesEntry" - json_name: "files" - } - nested_type { - name: "FilesEntry" - field { - name: "key" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "key" - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_BYTES - json_name: "value" - } - options { - map_entry: true - } - } - } - message_type { - name: "CreateResponse" - field { - name: "bundle" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "bundle" - } - } - service { - name: "Bundle" - method { - name: "Create" - input_type: ".containerd.vminitd.services.bundle.v1.CreateRequest" - output_type: ".containerd.vminitd.services.bundle.v1.CreateResponse" - } - } - options { - go_package: "github.com/spin-stack/spinbox/api/services/bundle/v1;bundle" - } - syntax: "proto3" -} -file { - name: "github.com/spin-stack/spinbox/api/services/stdio/v1/stdio.proto" - package: "containerd.vminitd.services.stdio.v1" - message_type { - name: "WriteStdinRequest" - field { - name: "container_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "containerId" - } - field { - name: "exec_id" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "execId" - } - field { - name: "data" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_BYTES - json_name: "data" - } - } - message_type { - name: "WriteStdinResponse" - field { - name: "bytes_written" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "bytesWritten" - } - } - message_type { - name: "ReadOutputRequest" - field { - name: "container_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "containerId" - } - field { - name: "exec_id" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "execId" - } - } - message_type { - name: "OutputChunk" - field { - name: "data" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_BYTES - json_name: "data" - } - field { - name: "eof" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_BOOL - json_name: "eof" - } - } - message_type { - name: "CloseStdinRequest" - field { - name: "container_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "containerId" - } - field { - name: "exec_id" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "execId" - } - } - message_type { - name: "CloseStdinResponse" - } - service { - name: "StdIO" - method { - name: "WriteStdin" - input_type: ".containerd.vminitd.services.stdio.v1.WriteStdinRequest" - output_type: ".containerd.vminitd.services.stdio.v1.WriteStdinResponse" - } - method { - name: "ReadStdout" - input_type: ".containerd.vminitd.services.stdio.v1.ReadOutputRequest" - output_type: ".containerd.vminitd.services.stdio.v1.OutputChunk" - server_streaming: true - } - method { - name: "ReadStderr" - input_type: ".containerd.vminitd.services.stdio.v1.ReadOutputRequest" - output_type: ".containerd.vminitd.services.stdio.v1.OutputChunk" - server_streaming: true - } - method { - name: "CloseStdin" - input_type: ".containerd.vminitd.services.stdio.v1.CloseStdinRequest" - output_type: ".containerd.vminitd.services.stdio.v1.CloseStdinResponse" - } - } - options { - go_package: "github.com/spin-stack/spinbox/api/services/stdio/v1;stdio" - } - syntax: "proto3" -} -file { - name: "google/protobuf/empty.proto" - package: "google.protobuf" - message_type { - name: "Empty" - } - options { - java_package: "com.google.protobuf" - java_outer_classname: "EmptyProto" - java_multiple_files: true - go_package: "google.golang.org/protobuf/types/known/emptypb" - cc_enable_arenas: true - objc_class_prefix: "GPB" - csharp_namespace: "Google.Protobuf.WellKnownTypes" - } - syntax: "proto3" -} -file { - name: "github.com/spin-stack/spinbox/api/services/system/v1/info.proto" - package: "containerd.vminitd.services.system.v1" - dependency: "google/protobuf/empty.proto" - message_type { - name: "InfoResponse" - field { - name: "version" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "version" - } - field { - name: "kernel_version" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "kernelVersion" - } - } - message_type { - name: "OfflineCPURequest" - field { - name: "cpu_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "cpuId" - } - } - message_type { - name: "OnlineCPURequest" - field { - name: "cpu_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "cpuId" - } - } - message_type { - name: "OfflineMemoryRequest" - field { - name: "memory_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "memoryId" - } - } - message_type { - name: "OnlineMemoryRequest" - field { - name: "memory_id" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "memoryId" - } - } - service { - name: "System" - method { - name: "Info" - input_type: ".google.protobuf.Empty" - output_type: ".containerd.vminitd.services.system.v1.InfoResponse" - } - method { - name: "OfflineCPU" - input_type: ".containerd.vminitd.services.system.v1.OfflineCPURequest" - output_type: ".google.protobuf.Empty" - } - method { - name: "OnlineCPU" - input_type: ".containerd.vminitd.services.system.v1.OnlineCPURequest" - output_type: ".google.protobuf.Empty" - } - method { - name: "OfflineMemory" - input_type: ".containerd.vminitd.services.system.v1.OfflineMemoryRequest" - output_type: ".google.protobuf.Empty" - } - method { - name: "OnlineMemory" - input_type: ".containerd.vminitd.services.system.v1.OnlineMemoryRequest" - output_type: ".google.protobuf.Empty" - } - } - options { - go_package: "github.com/spin-stack/spinbox/api/services/system/v1;system" - } - syntax: "proto3" -} -file { - name: "github.com/containerd/containerd/api/types/fieldpath.proto" - package: "containerd.types" - dependency: "google/protobuf/descriptor.proto" - extension { - name: "fieldpath_all" - extendee: ".google.protobuf.FileOptions" - number: 63300 - label: LABEL_OPTIONAL - type: TYPE_BOOL - json_name: "fieldpathAll" - proto3_optional: true - } - extension { - name: "fieldpath" - extendee: ".google.protobuf.MessageOptions" - number: 64400 - label: LABEL_OPTIONAL - type: TYPE_BOOL - json_name: "fieldpath" - proto3_optional: true - } - options { - go_package: "github.com/containerd/containerd/api/types;types" - } - syntax: "proto3" -} -file { - name: "google/protobuf/any.proto" - package: "google.protobuf" - message_type { - name: "Any" - field { - name: "type_url" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "typeUrl" - } - field { - name: "value" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_BYTES - json_name: "value" - } - } - options { - java_package: "com.google.protobuf" - java_outer_classname: "AnyProto" - java_multiple_files: true - go_package: "google.golang.org/protobuf/types/known/anypb" - objc_class_prefix: "GPB" - csharp_namespace: "Google.Protobuf.WellKnownTypes" - } - syntax: "proto3" -} -file { - name: "google/protobuf/timestamp.proto" - package: "google.protobuf" - message_type { - name: "Timestamp" - field { - name: "seconds" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_INT64 - json_name: "seconds" - } - field { - name: "nanos" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_INT32 - json_name: "nanos" - } - } - options { - java_package: "com.google.protobuf" - java_outer_classname: "TimestampProto" - java_multiple_files: true - go_package: "google.golang.org/protobuf/types/known/timestamppb" - cc_enable_arenas: true - objc_class_prefix: "GPB" - csharp_namespace: "Google.Protobuf.WellKnownTypes" - } - syntax: "proto3" -} -file { - name: "github.com/containerd/containerd/api/types/event.proto" - package: "containerd.types" - dependency: "github.com/containerd/containerd/api/types/fieldpath.proto" - dependency: "google/protobuf/any.proto" - dependency: "google/protobuf/timestamp.proto" - message_type { - name: "Envelope" - field { - name: "timestamp" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".google.protobuf.Timestamp" - json_name: "timestamp" - } - field { - name: "namespace" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "namespace" - } - field { - name: "topic" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "topic" - } - field { - name: "event" - number: 4 - label: LABEL_OPTIONAL - type: TYPE_MESSAGE - type_name: ".google.protobuf.Any" - json_name: "event" - } - options { - 64400: 1 - } - } - options { - go_package: "github.com/containerd/containerd/api/types;types" - } - syntax: "proto3" -} -file { - name: "github.com/spin-stack/spinbox/api/services/vmevents/v1/events.proto" - package: "spinbox.services.vmevents.v1" - dependency: "github.com/containerd/containerd/api/types/event.proto" - dependency: "google/protobuf/empty.proto" - service { - name: "Events" - method { - name: "Stream" - input_type: ".google.protobuf.Empty" - output_type: ".containerd.types.Envelope" - server_streaming: true - } - } - options { - go_package: "github.com/spin-stack/spinbox/api/services/vmevents/v1;vmevents" - } - syntax: "proto3" -} -file { - name: "github.com/spin-stack/spinbox/api/types/spinbox/v1/options.proto" - package: "io.containerd.spinbox.v1" - message_type { - name: "SpinboxOpts" - field { - name: "boot_cpus" - number: 1 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "bootCpus" - } - field { - name: "max_cpus" - number: 2 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "maxCpus" - } - field { - name: "memory_mb" - number: 3 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "memoryMb" - } - field { - name: "max_memory_mb" - number: 4 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "maxMemoryMb" - } - field { - name: "io_uid" - number: 7 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "ioUid" - } - field { - name: "io_gid" - number: 8 - label: LABEL_OPTIONAL - type: TYPE_UINT32 - json_name: "ioGid" - } - } - options { - go_package: "github.com/spin-stack/spinbox/api/types/spinbox/v1;spinbox" - } - syntax: "proto3" -} diff --git a/api/services/system/v1/info.proto b/api/services/system/v1/info.proto index dc9b207..50aaf52 100644 --- a/api/services/system/v1/info.proto +++ b/api/services/system/v1/info.proto @@ -16,6 +16,11 @@ service System { // This is called during VM initialization to verify the guest is ready. rpc Info(google.protobuf.Empty) returns (InfoResponse); + // PrepareShutdown performs pre-shutdown filesystem cleanup and sync. + // The host shim calls this before initiating VM poweroff so external snapshot + // and commit flows can rely on a quiesced writable layer. + rpc PrepareShutdown(google.protobuf.Empty) returns (google.protobuf.Empty); + // OfflineCPU takes a CPU offline via sysfs before hot-unplug. // The CPU must have been previously onlined and cannot be CPU 0 (boot CPU). // diff --git a/api/services/system/v1/info_ttrpc.pb.go b/api/services/system/v1/info_ttrpc.pb.go index 66b6d1a..e2f6cd6 100644 --- a/api/services/system/v1/info_ttrpc.pb.go +++ b/api/services/system/v1/info_ttrpc.pb.go @@ -10,6 +10,7 @@ import ( type TTRPCSystemService interface { Info(context.Context, *emptypb.Empty) (*InfoResponse, error) + PrepareShutdown(context.Context, *emptypb.Empty) (*emptypb.Empty, error) OfflineCPU(context.Context, *OfflineCPURequest) (*emptypb.Empty, error) OnlineCPU(context.Context, *OnlineCPURequest) (*emptypb.Empty, error) OfflineMemory(context.Context, *OfflineMemoryRequest) (*emptypb.Empty, error) @@ -26,6 +27,13 @@ func RegisterTTRPCSystemService(srv *ttrpc.Server, svc TTRPCSystemService) { } return svc.Info(ctx, &req) }, + "PrepareShutdown": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req emptypb.Empty + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.PrepareShutdown(ctx, &req) + }, "OfflineCPU": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { var req OfflineCPURequest if err := unmarshal(&req); err != nil { @@ -76,6 +84,14 @@ func (c *ttrpcsystemClient) Info(ctx context.Context, req *emptypb.Empty) (*Info return &resp, nil } +func (c *ttrpcsystemClient) PrepareShutdown(ctx context.Context, req *emptypb.Empty) (*emptypb.Empty, error) { + var resp emptypb.Empty + if err := c.client.Call(ctx, "containerd.vminitd.services.system.v1.System", "PrepareShutdown", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + func (c *ttrpcsystemClient) OfflineCPU(ctx context.Context, req *OfflineCPURequest) (*emptypb.Empty, error) { var resp emptypb.Empty if err := c.client.Call(ctx, "containerd.vminitd.services.system.v1.System", "OfflineCPU", req, &resp); err != nil { diff --git a/go.mod b/go.mod index c9976cc..e1f6a55 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/spin-stack/spinbox -go 1.25 +go 1.26 require ( github.com/containerd/cgroups/v3 v3.1.2 github.com/containerd/console v1.0.5 github.com/containerd/containerd/api v1.10.0 - github.com/containerd/containerd/v2 v2.2.1 + github.com/containerd/containerd/v2 v2.2.2 github.com/containerd/errdefs v1.0.0 github.com/containerd/errdefs/pkg v0.3.0 github.com/containerd/fifo v1.1.0 @@ -27,7 +27,7 @@ require ( github.com/vishvananda/netlink v1.3.1 github.com/vishvananda/netns v0.0.5 golang.org/x/sys v0.38.0 - google.golang.org/grpc v1.76.0 + google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.10 ) @@ -42,7 +42,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-events v0.0.0-20250808211157-605354379745 // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -66,16 +66,16 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sync v0.18.0 // indirect golang.org/x/text v0.31.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.2 // indirect ) diff --git a/go.sum b/go.sum index 0f50d2f..a1182b2 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd/api v1.10.0 h1:5n0oHYVBwN4VhoX9fFykCV9dF1/BvAXeg2F8W6UYq1o= github.com/containerd/containerd/api v1.10.0/go.mod h1:NBm1OAk8ZL+LG8R0ceObGxT5hbUYj7CzTmR3xh0DlMM= -github.com/containerd/containerd/v2 v2.2.1 h1:TpyxcY4AL5A+07dxETevunVS5zxqzuq7ZqJXknM11yk= -github.com/containerd/containerd/v2 v2.2.1/go.mod h1:NR70yW1iDxe84F2iFWbR9xfAN0N2F0NcjTi1OVth4nU= +github.com/containerd/containerd/v2 v2.2.2 h1:mjVQdtfryzT7lOqs5EYUFZm8ioPVjOpkSoG1GJPxEMY= +github.com/containerd/containerd/v2 v2.2.2/go.mod h1:5Jhevmv6/2J+Iu/A2xXAdUIdI5Ah/hfyO7okJ4AFIdY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -58,8 +58,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-events v0.0.0-20250808211157-605354379745 h1:yOn6Ze6IbYI/KAw2lw/83ELYvZh6hvsygTVkD0dzMC4= -github.com/docker/go-events v0.0.0-20250808211157-605354379745/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -161,8 +161,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -186,20 +186,20 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -275,15 +275,15 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/hack/release b/hack/release index 052ce7d..d7c2469 100755 --- a/hack/release +++ b/hack/release @@ -142,7 +142,7 @@ chmod +x "${RELEASE_DIR}/usr/share/spin-stack/libexec/cni/tc-redirect-tap" echo " ok: tc-redirect-tap CNI plugin" echo "Downloading spin-erofs-snapshotter and erofs tools..." -EROFS_SNAPSHOTTER_VERSION="${EROFS_SNAPSHOTTER_VERSION:-v20260117.06}" +EROFS_SNAPSHOTTER_VERSION="${EROFS_SNAPSHOTTER_VERSION:-v20260419.01}" EROFS_SNAPSHOTTER_BASE_URL="https://github.com/spin-stack/erofs-snapshotter/releases/download/${EROFS_SNAPSHOTTER_VERSION}" EROFS_SNAPSHOTTER_BINARIES=( diff --git a/internal/guest/services/system.go b/internal/guest/services/system.go index f9ee1b6..a4839a8 100644 --- a/internal/guest/services/system.go +++ b/internal/guest/services/system.go @@ -23,6 +23,7 @@ import ( emptypb "google.golang.org/protobuf/types/known/emptypb" api "github.com/spin-stack/spinbox/api/services/system/v1" + guestsystem "github.com/spin-stack/spinbox/internal/guest/vminit/system" "github.com/spin-stack/spinbox/internal/version" ) @@ -42,6 +43,8 @@ const ( type systemService struct{} +var prepareShutdown = guestsystem.Cleanup + var _ api.TTRPCSystemService = &systemService{} func init() { @@ -216,6 +219,11 @@ func (s *systemService) Info(ctx context.Context, _ *emptypb.Empty) (*api.InfoRe }, nil } +func (s *systemService) PrepareShutdown(ctx context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { + prepareShutdown(ctx) + return &emptypb.Empty{}, nil +} + func (s *systemService) OfflineCPU(ctx context.Context, req *api.OfflineCPURequest) (*emptypb.Empty, error) { cpuID := req.GetCpuID() if cpuID == 0 { diff --git a/internal/guest/services/system_test.go b/internal/guest/services/system_test.go index 7b2b82e..363571f 100644 --- a/internal/guest/services/system_test.go +++ b/internal/guest/services/system_test.go @@ -189,6 +189,33 @@ func TestSystemServiceInfo(t *testing.T) { }) } +func TestSystemServicePrepareShutdown(t *testing.T) { + svc := &systemService{} + + called := make(chan struct{}, 1) + original := prepareShutdown + prepareShutdown = func(context.Context) { + called <- struct{}{} + } + defer func() { + prepareShutdown = original + }() + + resp, err := svc.PrepareShutdown(context.Background(), &emptypb.Empty{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if resp == nil { + t.Fatal("expected non-nil response") + } + + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("prepare shutdown hook was not called") + } +} + func TestSystemServiceOfflineCPU(t *testing.T) { t.Run("reject CPU 0", func(t *testing.T) { svc := &systemService{} diff --git a/internal/host/vm/qemu/shutdown.go b/internal/host/vm/qemu/shutdown.go index 87234bc..328baf3 100644 --- a/internal/host/vm/qemu/shutdown.go +++ b/internal/host/vm/qemu/shutdown.go @@ -11,6 +11,9 @@ import ( "time" "github.com/containerd/log" + emptypb "google.golang.org/protobuf/types/known/emptypb" + + systemAPI "github.com/spin-stack/spinbox/api/services/system/v1" ) // Shutdown timing constants. @@ -19,6 +22,10 @@ const ( // shutdownQMPTimeout is the timeout for QMP commands during shutdown. shutdownQMPTimeout = 2 * time.Second + // shutdownPrepareGuestTimeout bounds the time we wait for the guest to flush + // filesystem state before starting poweroff. + shutdownPrepareGuestTimeout = 5 * time.Second + // shutdownACPIWait is how long to wait for guest to receive ACPI signal // before sending the quit command. shutdownACPIWait = 500 * time.Millisecond @@ -61,6 +68,24 @@ func (q *Instance) cleanupAfterFailedKill() { q.closeTAPFiles() } +func (q *Instance) prepareGuestShutdown(ctx context.Context, logger *log.Entry) { + if q.client == nil { + return + } + + logger.Info("qemu: requesting guest shutdown preparation") + prepareCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), shutdownPrepareGuestTimeout) + defer cancel() + + client := systemAPI.NewTTRPCSystemClient(q.client) + if _, err := client.PrepareShutdown(prepareCtx, &emptypb.Empty{}); err != nil { + logger.WithError(err).Warn("qemu: guest shutdown preparation failed") + return + } + + logger.Info("qemu: guest shutdown preparation completed") +} + func (q *Instance) stopQemuProcess(ctx context.Context, logger *log.Entry) error { // Brief wait to let guest start shutdown, then send quit // QEMU won't exit on its own - it always needs an explicit quit command @@ -239,6 +264,7 @@ func (q *Instance) Shutdown(ctx context.Context) error { q.mu.Lock() defer q.mu.Unlock() + q.prepareGuestShutdown(ctx, logger) q.closeClientConnections(logger) q.shutdownGuest(ctx, logger) diff --git a/internal/shim/task/create.go b/internal/shim/task/create.go index a7b79d2..f2d88e0 100644 --- a/internal/shim/task/create.go +++ b/internal/shim/task/create.go @@ -402,6 +402,12 @@ func (s *service) finalizeCreate(ctx context.Context, state *createState, resp * // // On failure, cleanup.rollback() releases resources in reverse order (LIFO). func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (*taskAPI.CreateTaskResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{ "id": r.ID, "bundle": r.Bundle, diff --git a/internal/shim/task/service.go b/internal/shim/task/service.go index ec55b75..410df61 100644 --- a/internal/shim/task/service.go +++ b/internal/shim/task/service.go @@ -167,6 +167,7 @@ func NewTaskService(ctx context.Context, publisher shim.Publisher, sd shutdown.S stateMachine: lifecycle.NewStateMachine(), events: make(chan any, eventChannelBuffer), eventsDone: make(chan struct{}), + forwardDone: make(chan struct{}), vmExitCh: make(chan struct{}), cpuHotplugControllers: make(map[string]cpuhotplug.CPUHotplugController), memoryHotplugControllers: make(map[string]memhotplug.MemoryHotplugController), @@ -337,13 +338,16 @@ type service struct { events chan any // === Shutdown Coordination === - initiateShutdown func() // Callback to trigger shutdown service - eventsClosed atomic.Bool // True when events channel is closed - eventsDone chan struct{} // Closed before events channel to signal send() to stop - eventsCloseOnce sync.Once // Ensures events channel closed exactly once - shutdownSvc shutdown.Service - inflight atomic.Int64 // Count of in-flight RPC calls for graceful shutdown - exitFunc func(code int) // Exit function (default: os.Exit), injectable for testing + initiateShutdown func() // Callback to trigger shutdown service + eventsClosed atomic.Bool // True when events channel is closed + eventsDone chan struct{} // Closed before events channel to signal send() to stop + eventsCloseOnce sync.Once // Ensures events channel closed exactly once + forwardDone chan struct{} // Closed when the event forwarder goroutine exits + shutdownOnce sync.Once // Ensures shutdown/exit sequencing runs once + shutdownRequested atomic.Bool + shutdownSvc shutdown.Service + inflight atomic.Int64 // Count of in-flight RPC calls for graceful shutdown + exitFunc func(code int) // Exit function (default: os.Exit), injectable for testing // === VM Exit Watchdog === // vmExitCh is closed when the QEMU process exits. @@ -369,7 +373,29 @@ func (s *service) RegisterTTRPC(server *ttrpc.Server) error { return nil } +func (s *service) isShuttingDown() bool { + return s.shutdownRequested.Load() || s.stateMachine.IsShuttingDown() +} + +func (s *service) beginRPC(allowDuringShutdown bool) (func(), error) { + if !allowDuringShutdown && s.isShuttingDown() { + return nil, errgrpc.ToGRPCf(errdefs.ErrFailedPrecondition, "shim is shutting down") + } + + s.inflight.Add(1) + if !allowDuringShutdown && s.isShuttingDown() { + s.inflight.Add(-1) + return nil, errgrpc.ToGRPCf(errdefs.ErrFailedPrecondition, "shim is shutting down") + } + + return func() { + s.inflight.Add(-1) + }, nil +} + func (s *service) shutdown(ctx context.Context) error { + s.shutdownRequested.Store(true) + // Transition to ShuttingDown state s.stateMachine.ForceTransition(lifecycle.StateShuttingDown) @@ -694,6 +720,12 @@ func sleepWithJitter(base time.Duration, jitterFraction float64) { // Start a process. func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (*taskAPI.StartResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + startTime := time.Now() log.G(ctx).WithFields(log.Fields{ "id": r.ID, @@ -763,15 +795,7 @@ type deleteCleanup struct { func (s *service) cleanupOnDeleteFailure(ctx context.Context, id string) { s.stateMachine.SetIntentionalShutdown(true) - - // Use partial cleanup starting from IO shutdown (skip hotplug which may not be running) - phases := s.buildCleanupPhases(id) - orchestrator := lifecycle.NewCleanupOrchestrator(phases) - result := orchestrator.ExecutePartial(ctx, lifecycle.PhaseIOShutdown) - - if result.HasErrors() { - log.G(ctx).WithField("failed_phases", result.FailedPhases()).Warn("cleanup after delete failure had errors") - } + go s.requestShutdownAndExit(ctx, fmt.Sprintf("delete failed for %s", id)) } func (s *service) collectDeleteCleanup(r *taskAPI.DeleteRequest) deleteCleanup { @@ -872,8 +896,11 @@ func (s *service) runDeleteCleanup(ctx context.Context, r *taskAPI.DeleteRequest // Delete the initial process and container. func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAPI.DeleteResponse, error) { - s.inflight.Add(1) - defer s.inflight.Add(-1) + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() log.G(ctx).WithFields(log.Fields{"id": r.ID, "exec": r.ExecID}).Debug("delete task request") // Mark deletion in progress (only for container, not exec) @@ -931,6 +958,12 @@ func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAP // Exec an additional process inside the container. func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID, "exec": r.ExecID}).Debug("exec request") vmc, cleanup, err := s.getTaskClient(ctx) @@ -1024,6 +1057,12 @@ func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (*pty // ResizePty of a process. func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID, "exec": r.ExecID}).Debug("resize pty request") vmc, cleanup, err := s.getTaskClient(ctx) if err != nil { @@ -1035,6 +1074,12 @@ func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (* // State returns runtime state information for a process. func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI.StateResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + // Use snapshot pattern to safely access container state outside lock. // This prevents data races when container fields are modified concurrently. snap := s.getContainerSnapshot() @@ -1099,6 +1144,12 @@ func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI. // Pause the container. func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("pause request") // Pause is not supported in VM-based runtime. // True pause would require checkpointing CPU and memory state (e.g., QEMU snapshot or CRIU), @@ -1108,6 +1159,12 @@ func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.E // Resume the container. func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("resume request") // Resume is not supported in VM-based runtime. // Without checkpoint support, there is no paused state to resume from. @@ -1116,6 +1173,12 @@ func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (*ptypes // Kill a process with the provided signal. func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID, "exec": r.ExecID}).Debug("kill request") vmc, cleanup, err := s.getTaskClient(ctx) if err != nil { @@ -1127,6 +1190,12 @@ func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (*ptypes.Emp // Pids returns all pids inside the container. func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (*taskAPI.PidsResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("pids request") vmc, cleanup, err := s.getTaskClient(ctx) if err != nil { @@ -1138,6 +1207,12 @@ func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (*taskAPI.Pi // CloseIO of a process. func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID, "exec": r.ExecID, "stdin": r.Stdin}).Debug("close io request") // If stdin is being closed and we have an RPC forwarder, signal it to close stdin. @@ -1170,6 +1245,12 @@ func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (*ptyp // Checkpoint the container. func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("checkpoint request") // Checkpoint is not supported in VM-based runtime. // Would require CRIU or QEMU snapshot to save/restore process state. @@ -1178,6 +1259,12 @@ func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskReque // Update a running container. func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (*ptypes.Empty, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("update request") vmc, cleanup, err := s.getTaskClient(ctx) if err != nil { @@ -1189,6 +1276,12 @@ func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (*pt // Wait for a process to exit. func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (*taskAPI.WaitResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID, "exec": r.ExecID}).Debug("wait request") vmc, cleanup, err := s.getTaskClient(ctx) if err != nil { @@ -1200,6 +1293,12 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (*taskAPI.Wa // Connect returns shim information such as the shim's pid. func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (*taskAPI.ConnectResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + // Use snapshot pattern to safely access container state outside lock. snap := s.getContainerSnapshot() hasContainer := snap != nil && snap.id == r.ID @@ -1237,8 +1336,11 @@ func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (*task } func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (*ptypes.Empty, error) { - s.inflight.Add(1) - defer s.inflight.Add(-1) + done, err := s.beginRPC(true) + if err != nil { + return nil, err + } + defer done() log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("shutdown request") s.stateMachine.SetIntentionalShutdown(true) @@ -1254,6 +1356,12 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (*pt } func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (*taskAPI.StatsResponse, error) { + done, err := s.beginRPC(false) + if err != nil { + return nil, err + } + defer done() + log.G(ctx).WithFields(log.Fields{"id": r.ID}).Debug("stats request") vmc, cleanup, err := s.getTaskClient(ctx) if err != nil { @@ -1367,67 +1475,88 @@ func (s *service) send(evt any) { } func (s *service) requestShutdownAndExit(ctx context.Context, reason string) { - log.G(ctx).WithField("reason", reason).Info("shim shutdown requested") - - // Use injectable exit function, defaulting to os.Exit - exit := s.exitFunc - if exit == nil { - exit = os.Exit - } - - // Ensure network cleanup happens before exit, regardless of shutdown service state. - // This is critical for unexpected VM exits (e.g., halt inside VM) where the normal - // Delete() path may not run, leaving CNI allocations orphaned. - // Use getContainerID() to safely access containerID outside the lock. - containerID := s.getContainerID() - if containerID != "" { - env := &network.Environment{ID: containerID} - if err := s.networkManager.ReleaseNetworkResources(ctx, env); err != nil { - log.G(ctx).WithError(err).WithField("id", containerID).Warn("failed to release network resources during unexpected shutdown") - } else { - log.G(ctx).WithField("id", containerID).Info("released network resources during unexpected shutdown") + started := false + s.shutdownOnce.Do(func() { + started = true + s.shutdownRequested.Store(true) + s.stateMachine.ForceTransition(lifecycle.StateShuttingDown) + + log.G(ctx).WithField("reason", reason).Info("shim shutdown requested") + + // Use injectable exit function, defaulting to os.Exit + exit := s.exitFunc + if exit == nil { + exit = os.Exit } - } - if s.shutdownSvc == nil { - log.G(ctx).WithField("reason", reason).Warn("shutdown service missing; exiting immediately") - exit(0) - return - } + // Wait for in-flight requests to complete before triggering shutdown callbacks. + inflightTimeout := time.NewTimer(5 * time.Second) + inflightTicker := time.NewTicker(10 * time.Millisecond) + defer inflightTimeout.Stop() + defer inflightTicker.Stop() - s.shutdownSvc.Shutdown() + inflightWait: + for s.inflight.Load() > 0 { + select { + case <-inflightTimeout.C: + log.G(ctx).WithFields(log.Fields{ + "reason": reason, + "inflight": s.inflight.Load(), + }).Warn("shutdown waiting for in-flight requests timed out") + break inflightWait + case <-inflightTicker.C: + // Check again. + } + } + + if s.shutdownSvc == nil { + log.G(ctx).WithField("reason", reason).Warn("shutdown service missing; running cleanup inline") + if err := s.shutdown(context.WithoutCancel(ctx)); err != nil { + log.G(ctx).WithError(err).WithField("reason", reason).Warn("inline shutdown completed with errors") + } + s.waitForForwarder(reason) + log.G(ctx).WithField("reason", reason).Info("exiting shim after inline shutdown") + exit(0) + return + } - // Wait for in-flight requests to complete using a ticker (avoids Sleep anti-pattern) - inflightTimeout := time.NewTimer(5 * time.Second) - inflightTicker := time.NewTicker(10 * time.Millisecond) - defer inflightTimeout.Stop() - defer inflightTicker.Stop() + s.shutdownSvc.Shutdown() -inflightWait: - for s.inflight.Load() > 0 { select { - case <-inflightTimeout.C: - log.G(ctx).WithFields(log.Fields{ - "reason": reason, - "inflight": s.inflight.Load(), - }).Warn("shutdown waiting for in-flight requests timed out") - break inflightWait - case <-inflightTicker.C: - // Check again + case <-s.shutdownSvc.Done(): + case <-time.After(5 * time.Second): + log.G(ctx).WithField("reason", reason).Warn("shutdown timeout; exiting anyway") } + + s.waitForForwarder(reason) + log.G(ctx).WithField("reason", reason).Info("exiting shim after shutdown") + exit(0) + }) + + if !started { + log.G(ctx).WithField("reason", reason).Debug("shim shutdown already in progress") + } +} + +func (s *service) waitForForwarder(reason string) { + if s.forwardDone == nil { + return } select { - case <-s.shutdownSvc.Done(): + case <-s.forwardDone: case <-time.After(5 * time.Second): - log.G(ctx).WithField("reason", reason).Warn("shutdown timeout; exiting anyway") + log.L.WithField("reason", reason).Warn("timed out waiting for event forwarder to exit") } - - log.G(ctx).WithField("reason", reason).Info("exiting shim after shutdown") - exit(0) } func (s *service) forward(ctx context.Context, publisher shim.Publisher, ready chan<- struct{}) { + defer func() { + if s.forwardDone != nil { + close(s.forwardDone) + } + }() + ns, ok := namespaces.Namespace(ctx) if !ok || ns == "" { ns = defaultNamespace diff --git a/internal/shim/task/service_test.go b/internal/shim/task/service_test.go new file mode 100644 index 0000000..d107391 --- /dev/null +++ b/internal/shim/task/service_test.go @@ -0,0 +1,310 @@ +//go:build linux + +package task + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/containerd/containerd/v2/pkg/shutdown" + + "github.com/spin-stack/spinbox/internal/host/network" + "github.com/spin-stack/spinbox/internal/shim/lifecycle" +) + +type mockShutdownService struct { + callbacks []func(context.Context) error + done chan struct{} + calls atomic.Int32 + mu sync.Mutex +} + +func newMockShutdownService() *mockShutdownService { + return &mockShutdownService{ + done: make(chan struct{}), + } +} + +func (m *mockShutdownService) Shutdown() { + m.calls.Add(1) +} + +func (m *mockShutdownService) RegisterCallback(fn func(context.Context) error) { + m.mu.Lock() + defer m.mu.Unlock() + m.callbacks = append(m.callbacks, fn) +} + +func (m *mockShutdownService) Done() <-chan struct{} { + return m.done +} + +func (m *mockShutdownService) Err() error { + select { + case <-m.done: + return shutdown.ErrShutdown + default: + return nil + } +} + +type mockNetworkManager struct { + releaseCalls atomic.Int32 + closeCalls atomic.Int32 +} + +func (m *mockNetworkManager) Close() error { + m.closeCalls.Add(1) + return nil +} + +func (m *mockNetworkManager) EnsureNetworkResources(context.Context, *network.Environment) error { + return nil +} + +func (m *mockNetworkManager) ReleaseNetworkResources(context.Context, *network.Environment) error { + m.releaseCalls.Add(1) + return nil +} + +func (m *mockNetworkManager) Metrics() *network.Metrics { + return nil +} + +func newTestService() *service { + forwardDone := make(chan struct{}) + close(forwardDone) + + return &service{ + stateMachine: lifecycle.NewStateMachine(), + events: make(chan any), + eventsDone: make(chan struct{}), + forwardDone: forwardDone, + vmExitCh: make(chan struct{}), + connManager: NewConnectionManager(nil, nil), + networkManager: &mockNetworkManager{}, + exitFunc: func(int) {}, + shutdownSvc: newMockShutdownService(), + initiateShutdown: nil, + } +} + +func waitForCondition(t *testing.T, timeout time.Duration, fn func() bool, msg string) { + t.Helper() + + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if fn() { + return + } + time.Sleep(10 * time.Millisecond) + } + + t.Fatal(msg) +} + +func TestBeginRPCRejectsDuringShutdown(t *testing.T) { + s := newTestService() + s.shutdownRequested.Store(true) + + done, err := s.beginRPC(false) + if err == nil { + t.Fatal("expected beginRPC to reject new RPCs during shutdown") + } + if done != nil { + t.Fatal("expected nil completion function on rejected RPC") + } + if got := s.inflight.Load(); got != 0 { + t.Fatalf("expected inflight to stay at 0, got %d", got) + } + + done, err = s.beginRPC(true) + if err != nil { + t.Fatalf("expected beginRPC(true) to allow shutdown RPC, got %v", err) + } + done() + if got := s.inflight.Load(); got != 0 { + t.Fatalf("expected inflight to return to 0, got %d", got) + } +} + +func TestRequestShutdownAndExitWaitsForInflight(t *testing.T) { + s := newTestService() + shutdownSvc := newMockShutdownService() + s.shutdownSvc = shutdownSvc + + exitCh := make(chan int, 1) + s.exitFunc = func(code int) { + exitCh <- code + } + + s.inflight.Add(1) + + go s.requestShutdownAndExit(context.Background(), "test") + + select { + case <-shutdownSvc.done: + t.Fatal("shutdown should not complete before inflight requests drain") + case <-time.After(100 * time.Millisecond): + } + + if got := shutdownSvc.calls.Load(); got != 0 { + t.Fatalf("expected shutdown callbacks not to start yet, got %d calls", got) + } + + s.inflight.Add(-1) + + waitForCondition(t, time.Second, func() bool { + return shutdownSvc.calls.Load() == 1 + }, "shutdown was not triggered after inflight drained") + + close(shutdownSvc.done) + + select { + case code := <-exitCh: + if code != 0 { + t.Fatalf("expected exit code 0, got %d", code) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for shim exit") + } +} + +func TestCleanupOnDeleteFailureTriggersFullShutdown(t *testing.T) { + s := newTestService() + shutdownSvc := newMockShutdownService() + s.shutdownSvc = shutdownSvc + + exitCh := make(chan int, 1) + s.exitFunc = func(code int) { + exitCh <- code + } + + s.stateMachine.ForceTransition(lifecycle.StateDeleting) + s.inflight.Add(1) + + s.cleanupOnDeleteFailure(context.Background(), "ctr") + + waitForCondition(t, time.Second, func() bool { + return s.stateMachine.IsShuttingDown() + }, "delete failure did not move shim to shutting down state") + + if !s.stateMachine.IsIntentionalShutdown() { + t.Fatal("expected delete failure shutdown to be marked intentional") + } + + if got := shutdownSvc.calls.Load(); got != 0 { + t.Fatalf("expected shutdown to wait for inflight delete RPC, got %d calls", got) + } + + s.inflight.Add(-1) + + waitForCondition(t, time.Second, func() bool { + return shutdownSvc.calls.Load() == 1 + }, "shutdown was not triggered after delete failure") + + close(shutdownSvc.done) + + select { + case <-exitCh: + case <-time.After(time.Second): + t.Fatal("timed out waiting for shim exit after delete failure") + } +} + +func TestRequestShutdownAndExitDoesNotReleaseNetworkInline(t *testing.T) { + s := newTestService() + shutdownSvc := newMockShutdownService() + networkManager := &mockNetworkManager{} + s.shutdownSvc = shutdownSvc + s.networkManager = networkManager + s.containerID = "ctr" + + exitCh := make(chan int, 1) + s.exitFunc = func(code int) { + exitCh <- code + } + + go s.requestShutdownAndExit(context.Background(), "vm process exit") + + waitForCondition(t, time.Second, func() bool { + return shutdownSvc.calls.Load() == 1 + }, "shutdown was not triggered") + + if got := networkManager.releaseCalls.Load(); got != 0 { + t.Fatalf("expected no inline network cleanup before shutdown callbacks, got %d releases", got) + } + + close(shutdownSvc.done) + + select { + case <-exitCh: + case <-time.After(time.Second): + t.Fatal("timed out waiting for shim exit") + } +} + +func TestRequestShutdownAndExitWaitsForForwarder(t *testing.T) { + s := newTestService() + shutdownSvc := newMockShutdownService() + s.shutdownSvc = shutdownSvc + s.forwardDone = make(chan struct{}) + + exitCh := make(chan int, 1) + s.exitFunc = func(code int) { + exitCh <- code + } + + go s.requestShutdownAndExit(context.Background(), "test") + + waitForCondition(t, time.Second, func() bool { + return shutdownSvc.calls.Load() == 1 + }, "shutdown was not triggered") + + close(shutdownSvc.done) + + select { + case <-exitCh: + t.Fatal("exit should wait for forwarder completion") + case <-time.After(100 * time.Millisecond): + } + + close(s.forwardDone) + + select { + case <-exitCh: + case <-time.After(time.Second): + t.Fatal("timed out waiting for exit after forwarder completion") + } +} + +func TestShutdownReleasesNetworkOnce(t *testing.T) { + s := newTestService() + networkManager := &mockNetworkManager{} + s.networkManager = networkManager + s.containerID = "ctr" + s.container = &container{} + + if err := s.shutdown(context.Background()); err != nil { + t.Fatalf("shutdown returned error: %v", err) + } + + if got := networkManager.releaseCalls.Load(); got != 1 { + t.Fatalf("expected one network release during shutdown, got %d", got) + } + if got := networkManager.closeCalls.Load(); got != 1 { + t.Fatalf("expected network manager close to be called once, got %d", got) + } + if !s.stateMachine.IsShuttingDown() { + t.Fatal("expected shim state to be shutting down") + } + + select { + case <-s.eventsDone: + default: + t.Fatal("expected eventsDone to be closed during shutdown") + } +}