diff --git a/.gitignore b/.gitignore index 9dffc16..81c9a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ Thumbs.db # kubebuilder test assets are cached locally # but shouldn't be committed bin/k8s/ +.council/ diff --git a/cmd/main.go b/cmd/main.go index 0ef6dd8..ed67e56 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -158,6 +158,7 @@ func main() { GatewayPublicDomain: os.Getenv("SEI_GATEWAY_PUBLIC_DOMAIN"), KubeRBACProxyImage: os.Getenv("SEI_KUBE_RBAC_PROXY_IMAGE"), SidecarImage: os.Getenv("SEI_SIDECAR_IMAGE"), + CosmosExporterImage: os.Getenv("SEI_COSMOS_EXPORTER_IMAGE"), } if err := platformCfg.Validate(); err != nil { diff --git a/internal/noderesource/noderesource.go b/internal/noderesource/noderesource.go index c54fbbf..0dd68e1 100644 --- a/internal/noderesource/noderesource.go +++ b/internal/noderesource/noderesource.go @@ -26,6 +26,18 @@ const ( // NodeLabel is the standard label key used on all SeiNode-owned resources. NodeLabel = "sei.io/node" + // chainLabel and roleLabel are observability identity labels lifted + // into `chain_id` and `component` metric labels by platform-owned + // (Pod|Service)Monitor relabelings. Pod-template only, never in the + // StatefulSet selector. + chainLabel = "sei.io/chain" + roleLabel = "sei.io/role" + + roleValidator = "validator" + roleArchive = "archive" + roleReplayer = "replayer" + roleFullNode = "node" + dataDir = platform.DataDir // homeVarRef is the K8s VariableReference form of HOME, substituted from @@ -42,15 +54,16 @@ const ( // Pod-spec container names. Used as both the .Name on built containers // and the lookup key for the operator-keyring containment guard. - containerNameSeid = "seid" - containerNameSidecar = "sei-sidecar" - containerNameRBACProxy = "kube-rbac-proxy" - servicePortNameAPI = "api" - rbacProxyConfigVolumeName = "rbac-proxy-config" - sidecarTLSVolumeName = "sidecar-tls" - rbacProxyConfigMountPath = "/etc/kube-rbac-proxy" - sidecarTLSMountPath = "/etc/tls" - RBACProxyPort int32 = 8443 + containerNameSeid = "seid" + containerNameSidecar = "sei-sidecar" + containerNameRBACProxy = "kube-rbac-proxy" + containerNameCosmosExporter = "cosmos-exporter" + servicePortNameAPI = "api" + rbacProxyConfigVolumeName = "rbac-proxy-config" + sidecarTLSVolumeName = "sidecar-tls" + rbacProxyConfigMountPath = "/etc/kube-rbac-proxy" + sidecarTLSMountPath = "/etc/tls" + RBACProxyPort int32 = 8443 pathHealthz = "/v0/healthz" pathLivez = "/v0/livez" @@ -73,11 +86,16 @@ const ( // sidecarTmpVolumeName backs an emptyDir at /tmp — required because the // sidecar runs with ReadOnlyRootFilesystem and Go stdlib defaults to /tmp. sidecarTmpVolumeName = "sidecar-tmp" + sidecarTmpMountPath = "/tmp" // sidecarNonRootUID is the nonroot UID/GID baked into distroless and // chainguard static-debian12 base images. Pod-level fsGroup matches so // the non-root sidecar can read kubelet-projected 0o400 Secret files. sidecarNonRootUID int64 = 65532 + + // defaultCosmosExporterPort matches sei-cosmos-exporter's upstream + // default. Platform PodMonitors target the named port `cosmos-metrics`. + defaultCosmosExporterPort int32 = 9300 ) // PlatformConfig is an alias for platform.Config. @@ -101,15 +119,36 @@ func SelectorLabels(node *seiv1alpha1.SeiNode) map[string]string { } // ResourceLabels returns labels for the StatefulSet pod template. -// User-provided podLabels are applied first; the system sei.io/node label -// is set last so it cannot be overridden. +// User-provided podLabels are applied first; system labels win. func ResourceLabels(node *seiv1alpha1.SeiNode) map[string]string { - labels := make(map[string]string, len(node.Spec.PodLabels)+1) + labels := make(map[string]string, len(node.Spec.PodLabels)+3) maps.Copy(labels, node.Spec.PodLabels) labels[NodeLabel] = node.Name + if node.Spec.ChainID != "" { + labels[chainLabel] = node.Spec.ChainID + } + if role := deriveRole(node); role != "" { + labels[roleLabel] = role + } return labels } +// deriveRole mirrors nodedeployment.deriveComponent so the pod label +// and the ServiceMonitor relabel-output stay in lock-step. +func deriveRole(node *seiv1alpha1.SeiNode) string { + switch { + case node.Spec.Validator != nil: + return roleValidator + case node.Spec.Archive != nil: + return roleArchive + case node.Spec.Replayer != nil: + return roleReplayer + case node.Spec.FullNode != nil: + return roleFullNode + } + return "" +} + // NodeMode returns the sei-config mode string for the node based on which // sub-spec is populated. Falls back to "full" if none is set. func NodeMode(node *seiv1alpha1.SeiNode) string { @@ -181,7 +220,10 @@ func GenerateStatefulSet(node *seiv1alpha1.SeiNode, p PlatformConfig) (*appsv1.S } one := int32(1) labels := ResourceLabels(node) - podSpec := buildNodePodSpec(node, p) + podSpec, err := buildNodePodSpec(node, p) + if err != nil { + return nil, err + } if err := assertNoOperatorKeyringOnSeidContainers(node, &podSpec); err != nil { return nil, err @@ -364,7 +406,7 @@ func ServicePorts() []corev1.ServicePort { // Internal helpers // --------------------------------------------------------------------------- -func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpec { +func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) (corev1.PodSpec, error) { dataVolume := corev1.Volume{ Name: "data", VolumeSource: corev1.VolumeSource{ @@ -444,9 +486,16 @@ func buildNodePodSpec(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.PodSpe initContainers = append(initContainers, buildRBACProxyContainer(node, p)) } spec.InitContainers = initContainers - spec.Containers = []corev1.Container{buildSidecarMainContainer(node, p)} + ceContainer, err := buildCosmosExporterContainer(p) + if err != nil { + return corev1.PodSpec{}, err + } + spec.Containers = []corev1.Container{ + buildSidecarMainContainer(node, p), + ceContainer, + } - return spec + return spec, nil } func sidecarImage(node *seiv1alpha1.SeiNode, p PlatformConfig) string { @@ -489,7 +538,7 @@ func buildSidecarContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) corev1.C mounts := make([]corev1.VolumeMount, 0, 2+len(keyringMounts)) mounts = append(mounts, corev1.VolumeMount{Name: "data", MountPath: dataDir}, - corev1.VolumeMount{Name: sidecarTmpVolumeName, MountPath: "/tmp"}, + corev1.VolumeMount{Name: sidecarTmpVolumeName, MountPath: sidecarTmpMountPath}, ) mounts = append(mounts, keyringMounts...) @@ -553,6 +602,62 @@ func buildSidecarMainContainer(node *seiv1alpha1.SeiNode, p PlatformConfig) core return container } +// defaultCosmosExporterResources: no CPU limit — cosmos-exporter calls +// seid's gRPC on every scrape; throttling turns into visible scrape gaps. +func defaultCosmosExporterResources() corev1.ResourceRequirements { + return corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("384Mi"), + }, + } +} + +// buildCosmosExporterContainer renders the cosmos-exporter sidecar. +// Image, args, port, and resources are fixed — no per-node knobs. +func buildCosmosExporterContainer(p PlatformConfig) (corev1.Container, error) { + if p.CosmosExporterImage == "" { + return corev1.Container{}, fmt.Errorf("SEI_COSMOS_EXPORTER_IMAGE is required on the operator Deployment") + } + return corev1.Container{ + Name: containerNameCosmosExporter, + Image: p.CosmosExporterImage, + Args: []string{ + "--denom", "usei", + "--denom-coefficient", "1000000", + "--bech-prefix", "sei", + "--listen-address", fmt.Sprintf(":%d", defaultCosmosExporterPort), + // --node and --tendermint-rpc default to localhost; the + // exporter shares the pod's net ns with seid. + }, + Ports: []corev1.ContainerPort{ + {Name: "cosmos-metrics", ContainerPort: defaultCosmosExporterPort, Protocol: corev1.ProtocolTCP}, + }, + SecurityContext: sidecarSecurityContext(), + Resources: defaultCosmosExporterResources(), + // /tmp: distroless + ReadOnlyRootFilesystem EROFS insurance. + VolumeMounts: []corev1.VolumeMount{ + {Name: sidecarTmpVolumeName, MountPath: sidecarTmpMountPath}, + }, + // cosmos-exporter Fatal()s on its initial gRPC dial. Gate + // startup on seid's gRPC port so we don't crash-loop until + // seid is up. + StartupProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt32(seiconfig.PortGRPC), + }, + }, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + FailureThreshold: 60, + }, + }, nil +} + func sidecarWaitCommand(node *seiv1alpha1.SeiNode) (command []string, args []string) { // Canonical seid invocation; spec.Entrypoint is silently ignored as of // HOME-based path resolution. "$HOME" (shell-expanded inside bash -c) diff --git a/internal/noderesource/noderesource_test.go b/internal/noderesource/noderesource_test.go index 07fb0de..709c58b 100644 --- a/internal/noderesource/noderesource_test.go +++ b/internal/noderesource/noderesource_test.go @@ -91,12 +91,18 @@ func mustGenerateStatefulSet(t *testing.T, node *seiv1alpha1.SeiNode, p Platform // --- Pod labels --- -func TestResourceLabelsForNode_DefaultsToNodeOnly(t *testing.T) { +func TestResourceLabelsForNode_DefaultsToSystemLabels(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") labels := ResourceLabels(node) - g.Expect(labels).To(Equal(map[string]string{NodeLabel: "snap-0"})) + // newSnapshotNode sets ChainID="sei-test" + FullNode mode, so chain + // + role labels are stamped alongside sei.io/node. + g.Expect(labels).To(Equal(map[string]string{ + NodeLabel: node.Name, + "sei.io/chain": "sei-test", + "sei.io/role": roleFullNode, + })) } func TestResourceLabelsForNode_MergesPodLabels(t *testing.T) { @@ -109,7 +115,9 @@ func TestResourceLabelsForNode_MergesPodLabels(t *testing.T) { labels := ResourceLabels(node) g.Expect(labels).To(Equal(map[string]string{ - NodeLabel: "snap-0", + NodeLabel: node.Name, + "sei.io/chain": "sei-test", + "sei.io/role": roleFullNode, "sei.io/nodedeployment": "my-group", "team": "platform", })) @@ -185,7 +193,8 @@ func TestBuildNodePodSpec_Genesis_MountsExistingPVC(t *testing.T) { g := NewWithT(t) node := newGenesisNode("mynet-0", "default") - spec := buildNodePodSpec(node, platformtest.Config()) + spec, err := buildNodePodSpec(node, platformtest.Config()) + g.Expect(err).NotTo(HaveOccurred()) g.Expect(spec.ServiceAccountName).To(Equal(platformtest.Config().ServiceAccount)) g.Expect(spec.Volumes).To(HaveLen(2)) // data PVC + sidecar-tmp emptyDir @@ -198,7 +207,8 @@ func TestBuildNodePodSpec_Snapshot_MountsNodePVC(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("snap-0", "default") - spec := buildNodePodSpec(node, platformtest.Config()) + spec, err := buildNodePodSpec(node, platformtest.Config()) + g.Expect(err).NotTo(HaveOccurred()) g.Expect(spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal("data-snap-0")) } @@ -718,7 +728,8 @@ func TestBuildNodePodSpec_Archive_SchedulesOnArchiveNodepool(t *testing.T) { g := NewWithT(t) node := newArchiveNode("archive-0", "pacific-1") - spec := buildNodePodSpec(node, platformtest.Config()) + spec, err := buildNodePodSpec(node, platformtest.Config()) + g.Expect(err).NotTo(HaveOccurred()) g.Expect(spec.Tolerations).To(HaveLen(1)) g.Expect(spec.Tolerations[0].Key).To(Equal("sei.io/workload")) @@ -735,7 +746,8 @@ func TestBuildNodePodSpec_FullNode_SchedulesOnDefaultNodepool(t *testing.T) { g := NewWithT(t) node := newSnapshotNode("syncer-0", "pacific-1") - spec := buildNodePodSpec(node, platformtest.Config()) + spec, err := buildNodePodSpec(node, platformtest.Config()) + g.Expect(err).NotTo(HaveOccurred()) g.Expect(spec.Tolerations[0].Value).To(Equal("sei-node")) @@ -1279,3 +1291,182 @@ func TestGenerateStatefulSet_ProductionPodSpec_PassesGuard(t *testing.T) { _, err := GenerateStatefulSet(node, platformtest.Config()) g.Expect(err).NotTo(HaveOccurred()) } + +// --- Cosmos exporter --- + +func TestCosmosExporter_AlwaysPresent(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + g.Expect(ce).NotTo(BeNil()) + g.Expect(sts.Spec.Template.Spec.Containers).To(HaveLen(2)) +} + +func TestCosmosExporter_DefaultImage(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + g.Expect(ce.Image).To(Equal(platformtest.Config().CosmosExporterImage)) +} + +func TestCosmosExporter_PortIsFixed(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + // Port is intentionally not user-configurable: the platform + // PodMonitor targets the named port `cosmos-metrics`. + g.Expect(ce.Ports[0].ContainerPort).To(Equal(int32(9300))) + g.Expect(ce.Ports[0].Name).To(Equal("cosmos-metrics")) +} + +func TestCosmosExporter_ErrorWhenImageUnset(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + cfg := platformtest.Config() + cfg.CosmosExporterImage = "" + + _, err := GenerateStatefulSet(node, cfg) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("SEI_COSMOS_EXPORTER_IMAGE is required")) +} + +func TestCosmosExporter_StartupProbeOnSeidGRPC(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + // Startup probe gates ListenAndServe on seid's gRPC being up so the + // exporter doesn't log.Fatal() on its initial dial. + g.Expect(ce.StartupProbe).NotTo(BeNil()) + g.Expect(ce.StartupProbe.TCPSocket).NotTo(BeNil()) + g.Expect(ce.StartupProbe.TCPSocket.Port.IntVal).To(Equal(int32(9090))) +} + +func TestCosmosExporter_MountsTmpEmptyDir(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + var hasTmp bool + for _, m := range ce.VolumeMounts { + if m.Name == sidecarTmpVolumeName && m.MountPath == sidecarTmpMountPath { + hasTmp = true + break + } + } + g.Expect(hasTmp).To(BeTrue(), "cosmos-exporter must mount sidecar-tmp at /tmp (ReadOnlyRootFilesystem)") +} + +func TestCosmosExporter_SeiArgs(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + g.Expect(ce.Args).To(ContainElements( + "--denom", "usei", + "--denom-coefficient", "1000000", + "--bech-prefix", "sei", + )) +} + +func TestCosmosExporter_DefaultResources(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + // 50m/64Mi requests, 256Mi memory limit, no CPU limit (see + // defaultCosmosExporterResources — scrape pulls would throttle). + cpuReq := ce.Resources.Requests[corev1.ResourceCPU] + memReq := ce.Resources.Requests[corev1.ResourceMemory] + memLim := ce.Resources.Limits[corev1.ResourceMemory] + g.Expect(cpuReq.String()).To(Equal("50m")) + g.Expect(memReq.String()).To(Equal("64Mi")) + g.Expect(memLim.String()).To(Equal("384Mi")) + _, hasCPULimit := ce.Resources.Limits[corev1.ResourceCPU] + g.Expect(hasCPULimit).To(BeFalse()) +} + +func TestResourceLabels_ChainAndRoleStampedUnconditionally(t *testing.T) { + g := NewWithT(t) + tests := []struct { + name string + mutate func(*seiv1alpha1.SeiNode) + expected string + }{ + {"validator", func(n *seiv1alpha1.SeiNode) { n.Spec.Validator = &seiv1alpha1.ValidatorSpec{} }, roleValidator}, + {"archive", func(n *seiv1alpha1.SeiNode) { n.Spec.Archive = &seiv1alpha1.ArchiveSpec{} }, roleArchive}, + {"fullNode", func(n *seiv1alpha1.SeiNode) { n.Spec.FullNode = &seiv1alpha1.FullNodeSpec{} }, roleFullNode}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "n", Namespace: "ns"}, + Spec: seiv1alpha1.SeiNodeSpec{ChainID: "pacific-1", Image: "ghcr.io/sei-protocol/seid:latest"}, + } + tt.mutate(node) + + labels := ResourceLabels(node) + + g.Expect(labels).To(HaveKeyWithValue("sei.io/chain", "pacific-1")) + g.Expect(labels).To(HaveKeyWithValue("sei.io/role", tt.expected)) + }) + } +} + +func TestResourceLabels_ChainOmittedWhenChainIDEmpty(t *testing.T) { + g := NewWithT(t) + node := &seiv1alpha1.SeiNode{ + ObjectMeta: metav1.ObjectMeta{Name: "n", Namespace: "ns"}, + Spec: seiv1alpha1.SeiNodeSpec{FullNode: &seiv1alpha1.FullNodeSpec{}}, + } + + labels := ResourceLabels(node) + + g.Expect(labels).NotTo(HaveKey("sei.io/chain")) +} + +func TestResourceLabels_NotInSelector(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + + // Chain + role must NOT live in the immutable StatefulSet selector + // — otherwise renaming the chain (rare) or rolling between modes + // would require StatefulSet recreation. Only sei.io/node belongs + // in the selector. + g.Expect(sts.Spec.Selector.MatchLabels).NotTo(HaveKey("sei.io/chain")) + g.Expect(sts.Spec.Selector.MatchLabels).NotTo(HaveKey("sei.io/role")) + g.Expect(sts.Spec.Selector.MatchLabels).To(HaveLen(1)) +} + +func TestCosmosExporter_NonRootSecurityContext(t *testing.T) { + g := NewWithT(t) + node := newSnapshotNode("ce-0", "default") + + sts := mustGenerateStatefulSet(t, node, platformtest.Config()) + ce := findContainer(sts.Spec.Template.Spec.Containers, containerNameCosmosExporter) + + g.Expect(ce.SecurityContext).NotTo(BeNil()) + g.Expect(ce.SecurityContext.RunAsNonRoot).NotTo(BeNil()) + g.Expect(*ce.SecurityContext.RunAsNonRoot).To(BeTrue()) + g.Expect(*ce.SecurityContext.RunAsUser).To(Equal(int64(65532))) +} diff --git a/internal/platform/platform.go b/internal/platform/platform.go index 713d2c6..a7f2976 100644 --- a/internal/platform/platform.go +++ b/internal/platform/platform.go @@ -51,6 +51,10 @@ type Config struct { KubeRBACProxyImage string SidecarImage string + + // CosmosExporterImage is the sei-cosmos-exporter sidecar image. + // Required when any SeiNode sets spec.cosmosExporter: true. + CosmosExporterImage string } // NodepoolForMode returns the Karpenter NodePool name for the given diff --git a/internal/platform/platformtest/config.go b/internal/platform/platformtest/config.go index 37de0ba..fdc97a4 100644 --- a/internal/platform/platformtest/config.go +++ b/internal/platform/platformtest/config.go @@ -33,6 +33,7 @@ func Config() platform.Config { KubeRBACProxyImage: "quay.io/brancz/kube-rbac-proxy:v0.19.1", // Arbitrary fixture; not authoritative. Production digest is set // via SEI_SIDECAR_IMAGE in the platform repo's controller Deployment. - SidecarImage: "ghcr.io/sei-protocol/seictl@sha256:a2af4e1b8ed4c12661a3c98cce050bae3f292cc7560abc2ba98fd7dfc80d9be5", + SidecarImage: "ghcr.io/sei-protocol/seictl@sha256:a2af4e1b8ed4c12661a3c98cce050bae3f292cc7560abc2ba98fd7dfc80d9be5", + CosmosExporterImage: "ghcr.io/sei-protocol/sei-cosmos-exporter@sha256:0000000000000000000000000000000000000000000000000000000000000000", } }