WIP: Add Application Load Balancer Controller Manager #879
+3,436
−11
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,333 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| "errors" | ||
| "flag" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) | ||
| // to ensure that exec-entrypoint and run can make use of them. | ||
| sdkconfig "github.com/stackitcloud/stackit-sdk-go/core/config" | ||
| "gopkg.in/yaml.v3" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| utilruntime "k8s.io/apimachinery/pkg/util/runtime" | ||
| clientgoscheme "k8s.io/client-go/kubernetes/scheme" | ||
| _ "k8s.io/client-go/plugin/pkg/client/auth" | ||
| ctrl "sigs.k8s.io/controller-runtime" | ||
| "sigs.k8s.io/controller-runtime/pkg/certwatcher" | ||
| "sigs.k8s.io/controller-runtime/pkg/healthz" | ||
| "sigs.k8s.io/controller-runtime/pkg/log/zap" | ||
| "sigs.k8s.io/controller-runtime/pkg/metrics/filters" | ||
| metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" | ||
| "sigs.k8s.io/controller-runtime/pkg/webhook" | ||
|
|
||
| "github.com/stackitcloud/cloud-provider-stackit/pkg/alb/ingress" | ||
| albclient "github.com/stackitcloud/cloud-provider-stackit/pkg/stackit" | ||
| albsdk "github.com/stackitcloud/stackit-sdk-go/services/alb/v2api" | ||
| certsdk "github.com/stackitcloud/stackit-sdk-go/services/certificates/v2api" | ||
| // +kubebuilder:scaffold:imports | ||
| ) | ||
|
|
||
| var ( | ||
| scheme = runtime.NewScheme() | ||
| setupLog = ctrl.Log.WithName("setup") | ||
| ) | ||
|
|
||
| func init() { | ||
| utilruntime.Must(clientgoscheme.AddToScheme(scheme)) | ||
|
|
||
| // +kubebuilder:scaffold:scheme | ||
| } | ||
|
|
||
| type Config struct { | ||
| NetworkID string `yaml:"networkID"` | ||
| ProjectID string `yaml:"projectID"` | ||
| Region string `yaml:"region"` | ||
| } | ||
|
|
||
| // ReadConfig reads the ALB infrastructure configuration provided via the cloud-config flag. | ||
| func ReadConfig(cloudConfig string) (Config, error) { | ||
| configFile, err := os.Open(cloudConfig) | ||
| if err != nil { | ||
| return Config{}, err | ||
| } | ||
| defer configFile.Close() | ||
|
|
||
| var config Config | ||
| content, err := io.ReadAll(configFile) | ||
| if err != nil { | ||
| return Config{}, err | ||
| } | ||
|
|
||
| err = yaml.Unmarshal(content, &config) | ||
| if err != nil { | ||
| return Config{}, err | ||
| } | ||
|
|
||
| if config.ProjectID == "" { | ||
| return Config{}, errors.New("project ID must be set") | ||
| } | ||
| if config.Region == "" { | ||
| return Config{}, errors.New("region must be set") | ||
| } | ||
| if config.NetworkID == "" { | ||
| return Config{}, errors.New("network ID must be set") | ||
| } | ||
| return config, nil | ||
| } | ||
|
|
||
| // nolint:gocyclo,funlen // TODO: Refactor into smaller functions. | ||
| func main() { | ||
| var metricsAddr string | ||
| var metricsCertPath, metricsCertName, metricsCertKey string | ||
| var webhookCertPath, webhookCertName, webhookCertKey string | ||
| var enableLeaderElection bool | ||
| var leaderElectionNamespace string | ||
| var leaderElectionID string | ||
| var probeAddr string | ||
| var cloudConfig string | ||
| var secureMetrics bool | ||
| var enableHTTP2 bool | ||
| var tlsOpts []func(*tls.Config) | ||
| flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ | ||
| "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") | ||
| flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") | ||
| flag.BoolVar(&enableLeaderElection, "leader-elect", false, | ||
| "Enable leader election for controller manager. "+ | ||
| "Enabling this will ensure there is only one active controller manager.") | ||
| flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "default", "The namespace in which the leader "+ | ||
| "election resource will be created.") | ||
| flag.StringVar(&leaderElectionID, "leader-election-id", "d0fbe9c4.stackit.cloud", "The name of the resource that "+ | ||
| "leader election will use for holding the leader lock.") | ||
| flag.BoolVar(&secureMetrics, "metrics-secure", true, | ||
| "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") | ||
| flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") | ||
| flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") | ||
| flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") | ||
| flag.StringVar(&metricsCertPath, "metrics-cert-path", "", | ||
| "The directory that contains the metrics server certificate.") | ||
| flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") | ||
| flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") | ||
| flag.BoolVar(&enableHTTP2, "enable-http2", false, | ||
| "If set, HTTP/2 will be enabled for the metrics and webhook servers") | ||
| flag.StringVar(&cloudConfig, "cloud-config", "cloud.yaml", "The path to the cloud config file.") | ||
| opts := zap.Options{ | ||
| Development: true, | ||
| } | ||
| opts.BindFlags(flag.CommandLine) | ||
| flag.Parse() | ||
|
|
||
| ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) | ||
|
|
||
| config, err := ReadConfig(cloudConfig) | ||
| if err != nil { | ||
| setupLog.Error(err, "Failed to read cloud config") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| // if the enable-http2 flag is false (the default), http/2 should be disabled | ||
| // due to its vulnerabilities. More specifically, disabling http/2 will | ||
| // prevent from being vulnerable to the HTTP/2 Stream Cancellation and | ||
| // Rapid Reset CVEs. For more information see: | ||
| // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 | ||
| // - https://github.com/advisories/GHSA-4374-p667-p6c8 | ||
| disableHTTP2 := func(c *tls.Config) { | ||
| setupLog.Info("disabling http/2") | ||
| c.NextProtos = []string{"http/1.1"} | ||
| } | ||
|
|
||
| if !enableHTTP2 { | ||
| tlsOpts = append(tlsOpts, disableHTTP2) | ||
| } | ||
|
|
||
| // Create watchers for metrics and webhooks certificates | ||
| var metricsCertWatcher, webhookCertWatcher *certwatcher.CertWatcher | ||
|
|
||
| // Initial webhook TLS options | ||
| webhookTLSOpts := tlsOpts | ||
|
|
||
| if webhookCertPath != "" { | ||
| setupLog.Info("Initializing webhook certificate watcher using provided certificates", | ||
| "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) | ||
|
|
||
| var err error | ||
| webhookCertWatcher, err = certwatcher.New( | ||
| filepath.Join(webhookCertPath, webhookCertName), | ||
| filepath.Join(webhookCertPath, webhookCertKey), | ||
| ) | ||
| if err != nil { | ||
| setupLog.Error(err, "Failed to initialize webhook certificate watcher") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| webhookTLSOpts = append(webhookTLSOpts, func(config *tls.Config) { | ||
| config.GetCertificate = webhookCertWatcher.GetCertificate | ||
| }) | ||
| } | ||
|
|
||
| webhookServer := webhook.NewServer(webhook.Options{ | ||
| TLSOpts: webhookTLSOpts, | ||
| }) | ||
|
|
||
| // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. | ||
| // More info: | ||
| // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.4/pkg/metrics/server | ||
| // - https://book.kubebuilder.io/reference/metrics.html | ||
| metricsServerOptions := metricsserver.Options{ | ||
| BindAddress: metricsAddr, | ||
| SecureServing: secureMetrics, | ||
| TLSOpts: tlsOpts, | ||
| } | ||
|
|
||
| if secureMetrics { | ||
| // FilterProvider is used to protect the metrics endpoint with authn/authz. | ||
| // These configurations ensure that only authorized users and service accounts | ||
| // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: | ||
| // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.4/pkg/metrics/filters#WithAuthenticationAndAuthorization | ||
| metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization | ||
| } | ||
|
|
||
| // If the certificate is not specified, controller-runtime will automatically | ||
| // generate self-signed certificates for the metrics server. While convenient for development and testing, | ||
| // this setup is not recommended for production. | ||
| // | ||
| // TODO(user): If you enable certManager, uncomment the following lines: | ||
| // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates | ||
| // managed by cert-manager for the metrics server. | ||
| // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. | ||
| if metricsCertPath != "" { | ||
| setupLog.Info("Initializing metrics certificate watcher using provided certificates", | ||
| "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) | ||
|
|
||
| var err error | ||
| metricsCertWatcher, err = certwatcher.New( | ||
| filepath.Join(metricsCertPath, metricsCertName), | ||
| filepath.Join(metricsCertPath, metricsCertKey), | ||
| ) | ||
| if err != nil { | ||
| setupLog.Error(err, "to initialize metrics certificate watcher", "error", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| metricsServerOptions.TLSOpts = append(metricsServerOptions.TLSOpts, func(config *tls.Config) { | ||
| config.GetCertificate = metricsCertWatcher.GetCertificate | ||
| }) | ||
| } | ||
|
|
||
| mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ | ||
| Scheme: scheme, | ||
| Metrics: metricsServerOptions, | ||
| WebhookServer: webhookServer, | ||
| HealthProbeBindAddress: probeAddr, | ||
| LeaderElection: enableLeaderElection, | ||
| LeaderElectionID: leaderElectionID, | ||
| LeaderElectionNamespace: leaderElectionNamespace, | ||
| // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily | ||
| // when the Manager ends. This requires the binary to immediately end when the | ||
| // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly | ||
| // speeds up voluntary leader transitions as the new leader don't have to wait | ||
| // LeaseDuration time first. | ||
| // | ||
| // In the default scaffold provided, the program ends immediately after | ||
| // the manager stops, so would be fine to enable this option. However, | ||
| // if you are doing or is intended to do any operation such as perform cleanups | ||
| // after the manager stops then its usage might be unsafe. | ||
| // LeaderElectionReleaseOnCancel: true, | ||
| }) | ||
| if err != nil { | ||
| setupLog.Error(err, "unable to start manager") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| albURL, _ := os.LookupEnv("STACKIT_LOAD_BALANCER_API_ALB_URL") | ||
|
|
||
| certURL, _ := os.LookupEnv("STACKIT_LOAD_BALANCER_API_CERT_URL") | ||
|
|
||
| // Create an ALB SDK client | ||
| albOpts := []sdkconfig.ConfigurationOption{} | ||
|
|
||
| if albURL != "" { | ||
| albOpts = append(albOpts, sdkconfig.WithEndpoint(albURL)) | ||
| } | ||
|
|
||
| certOpts := []sdkconfig.ConfigurationOption{} | ||
| if certURL != "" { | ||
| certOpts = append(certOpts, sdkconfig.WithEndpoint(certURL)) | ||
| } | ||
|
|
||
| fmt.Printf("Create ALB SDK client\n") | ||
|
There was a problem hiding this comment. please remove fmt.Printf or use setupLog instead |
||
| sdkClient, err := albsdk.NewAPIClient(albOpts...) | ||
| if err != nil { | ||
| setupLog.Error(err, "unable to create ALB SDK client", "controller", "IngressClass") | ||
| os.Exit(1) | ||
| } | ||
| // Create an ALB client | ||
| fmt.Printf("Create ALB client\n") | ||
| albClient, err := albclient.NewApplicationLoadBalancerClient(sdkClient) | ||
| if err != nil { | ||
| setupLog.Error(err, "unable to create ALB client", "controller", "IngressClass") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| // Create an Certificates SDK client | ||
| certificateAPI, err := certsdk.NewAPIClient(certOpts...) | ||
| if err != nil { | ||
| setupLog.Error(err, "unable to create certificate SDK client", "controller", "IngressClass") | ||
| os.Exit(1) | ||
| } | ||
| // Create an Certificates API client | ||
| certificateClient, err := albclient.NewCertClient(certificateAPI) | ||
| if err != nil { | ||
| setupLog.Error(err, "unable to create Certificates client", "controller", "IngressClass") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| if err = (&ingress.IngressClassReconciler{ | ||
| Client: mgr.GetClient(), | ||
| ALBClient: albClient, | ||
| CertificateClient: certificateClient, | ||
| Scheme: mgr.GetScheme(), | ||
| ProjectID: config.ProjectID, | ||
| NetworkID: config.NetworkID, | ||
| Region: config.Region, | ||
| }).SetupWithManager(mgr); err != nil { | ||
| setupLog.Error(err, "unable to create controller", "controller", "IngressClass") | ||
| os.Exit(1) | ||
| } | ||
| // +kubebuilder:scaffold:builder | ||
|
|
||
| if metricsCertWatcher != nil { | ||
| setupLog.Info("Adding metrics certificate watcher to manager") | ||
| if err := mgr.Add(metricsCertWatcher); err != nil { | ||
| setupLog.Error(err, "unable to add metrics certificate watcher to manager") | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| if webhookCertWatcher != nil { | ||
| setupLog.Info("Adding webhook certificate watcher to manager") | ||
| if err := mgr.Add(webhookCertWatcher); err != nil { | ||
| setupLog.Error(err, "unable to add webhook certificate watcher to manager") | ||
| os.Exit(1) | ||
| } | ||
| } | ||
|
|
||
| if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { | ||
| setupLog.Error(err, "unable to set up health check") | ||
| os.Exit(1) | ||
| } | ||
| if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { | ||
| setupLog.Error(err, "unable to set up ready check") | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| setupLog.Info("starting manager") | ||
| if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { | ||
| setupLog.Error(err, "problem running manager") | ||
| os.Exit(1) | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
might be good to create separate examples for that