From 95be85292cc3a8e4788ebc78146c4b4f192945a9 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 09:28:01 +0200 Subject: [PATCH 01/16] feat: add merged_overrides() to AirflowCluster Computes the merged env and webserver_config.py overrides directly from the CRD (role <- role-group), replacing the product-config override path. Co-Authored-By: Claude Opus 4.8 --- rust/operator-binary/src/crd/mod.rs | 138 +++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 83c54ef1..521a98f5 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; @@ -19,7 +19,7 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, - config_overrides::KeyValueConfigOverrides, + config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, crd::git_sync, deep_merger::ObjectOverrides, k8s_openapi::{ @@ -114,7 +114,7 @@ pub struct AirflowConfigOverrides { pub webserver_config_py: Option, } -impl stackable_operator::config_overrides::KeyValueOverridesProvider for AirflowConfigOverrides { +impl KeyValueOverridesProvider for AirflowConfigOverrides { fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { match file { AIRFLOW_CONFIG_FILENAME => self @@ -127,6 +127,12 @@ impl stackable_operator::config_overrides::KeyValueOverridesProvider for Airflow } } +#[derive(Clone, Debug)] +pub struct MergedOverrides { + pub env_overrides: HashMap, + pub config_file_overrides: BTreeMap, +} + #[derive(Snafu, Debug)] pub enum Error { #[snafu(display("Unknown Airflow role found {role}. Should be one of {roles:?}"))] @@ -450,6 +456,39 @@ impl v1alpha2::AirflowCluster { format!("{}-executor-pod-template", self.name_any()) } + pub fn merged_overrides( + &self, + role: &AirflowRole, + rolegroup_name: &str, + ) -> Result { + let role_config = role.role_config(self)?; + + let mut env_overrides = role_config.config.env_overrides.clone(); + let mut file_overrides = role_config + .config + .config_overrides + .get_key_value_overrides(AIRFLOW_CONFIG_FILENAME); + + if let Some(rg) = role_config.role_groups.get(rolegroup_name) { + env_overrides.extend(rg.config.env_overrides.clone()); + let rg_file = rg + .config + .config_overrides + .get_key_value_overrides(AIRFLOW_CONFIG_FILENAME); + file_overrides.extend(rg_file); + } + + let config_file_overrides = file_overrides + .into_iter() + .filter_map(|(k, v)| v.map(|v| (k, v))) + .collect(); + + Ok(MergedOverrides { + env_overrides, + config_file_overrides, + }) + } + /// Retrieve and merge resource configs for role and role groups pub fn merged_config( &self, @@ -1114,13 +1153,15 @@ pub fn build_recommended_labels<'a, T>( #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use indoc::formatdoc; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, versioned::test_utils::RoundtripTestData, }; - use crate::{v1alpha1, v1alpha2}; + use crate::{crd::AirflowRole, v1alpha1, v1alpha2}; #[test] fn test_cluster_config() { @@ -1345,4 +1386,93 @@ mod tests { "# } } + + #[test] + fn merged_overrides_match_config_overrides_from_crd() { + let cluster_yaml = r#" + apiVersion: airflow.stackable.tech/v1alpha2 + kind: AirflowCluster + metadata: + name: airflow + spec: + image: + productVersion: 3.1.6 + clusterConfig: + loadExamples: false + exposeConfig: false + credentialsSecretName: airflow-admin-credentials + metadataDatabase: + postgresql: + host: airflow-postgresql + database: airflow + credentialsSecretName: airflow-postgresql-credentials + webservers: + config: {} + configOverrides: + webserver_config.py: + AUTH_TYPE: "AUTH_OID" + ROLE_ONLY_KEY: "role-value" + envOverrides: + ROLE_ENV_VAR: "role-env-value" + roleGroups: + default: + config: {} + configOverrides: + webserver_config.py: + AUTH_TYPE: "AUTH_DB" + GROUP_ONLY_KEY: "group-value" + envOverrides: + GROUP_ENV_VAR: "group-env-value" + schedulers: + config: {} + roleGroups: + default: + config: {} + kubernetesExecutors: + config: {} + "#; + + let deserializer = serde_yaml::Deserializer::from_str(cluster_yaml); + let cluster: v1alpha2::AirflowCluster = + serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); + + // Webservers/default: role-group overrides merge with (and override) role-level ones + let overrides = cluster + .merged_overrides(&AirflowRole::Webserver, "default") + .expect("merged_overrides should succeed"); + + // configOverrides: group AUTH_TYPE overrides role AUTH_TYPE, both unique keys kept + assert_eq!( + overrides.config_file_overrides, + BTreeMap::from([ + ("AUTH_TYPE".into(), "AUTH_DB".into()), + ("ROLE_ONLY_KEY".into(), "role-value".into()), + ("GROUP_ONLY_KEY".into(), "group-value".into()), + ]) + ); + + // envOverrides: both role and group env vars present + assert_eq!(overrides.env_overrides.len(), 2); + assert_eq!( + overrides.env_overrides.get("ROLE_ENV_VAR").unwrap(), + "role-env-value" + ); + assert_eq!( + overrides.env_overrides.get("GROUP_ENV_VAR").unwrap(), + "group-env-value" + ); + + // Schedulers/default: no overrides configured → both maps empty + let overrides = cluster + .merged_overrides(&AirflowRole::Scheduler, "default") + .expect("merged_overrides should succeed"); + assert!( + overrides.config_file_overrides.is_empty(), + "scheduler should have no config file overrides" + ); + assert!( + overrides.env_overrides.is_empty(), + "scheduler should have no env overrides" + ); + } } From 5ff22fb7e7d39c38fdceb5dd722ca8f6a5ade704 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 10:57:18 +0200 Subject: [PATCH 02/16] refactor: remove product-config validation from reconcile Replaces transform_all_roles_to_config / validate_all_roles_and_groups_config with direct role iteration plus merged_overrides(). Folds the dereferenced authentication/authorization objects into ValidatedAirflowCluster so downstream build steps read them from the validated cluster. The product-config crate is still used for the Flask config writer (removed in a later step). Co-Authored-By: Claude Opus 4.8 --- .../operator-binary/src/airflow_controller.rs | 48 ++++---- .../src/controller/validate.rs | 112 +++++++----------- rust/operator-binary/src/crd/authorization.rs | 2 + rust/operator-binary/src/crd/mod.rs | 30 ----- rust/operator-binary/src/env_vars.rs | 26 ++-- rust/operator-binary/src/main.rs | 8 +- 6 files changed, 73 insertions(+), 153 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index ea7dd9ad..7e6112f4 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -6,11 +6,7 @@ use std::{ }; use const_format::concatcp; -use product_config::{ - ProductConfigManager, - flask_app_config_writer::{self, FlaskAppConfigWriterError}, - types::PropertyNameKind, -}; +use product_config::flask_app_config_writer::{self, FlaskAppConfigWriterError}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -61,10 +57,7 @@ use stackable_operator::{ }, kvp::{Annotation, Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, - product_config_utils::{ - CONFIG_OVERRIDE_FILE_FOOTER_KEY, CONFIG_OVERRIDE_FILE_HEADER_KEY, env_vars_from, - env_vars_from_rolegroup_config, - }, + product_config_utils::env_vars_from, product_logging::{ self, framework::LoggingError, @@ -118,9 +111,11 @@ pub const CONTAINER_IMAGE_BASE_NAME: &str = "airflow"; pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); +const CONFIG_OVERRIDE_FILE_HEADER_KEY: &str = "FILE_HEADER"; +const CONFIG_OVERRIDE_FILE_FOOTER_KEY: &str = "FILE_FOOTER"; + pub struct Ctx { pub client: stackable_operator::client::Client, - pub product_config: ProductConfigManager, pub operator_environment: OperatorEnvironmentOptions, } @@ -386,7 +381,7 @@ pub async fn reconcile_airflow( CONTAINER_IMAGE_BASE_NAME, &ctx.operator_environment.image_repository, crate::built_info::PKG_VERSION, - &ctx.product_config, + dereferenced, ) .context(ValidateSnafu)?; @@ -464,8 +459,8 @@ pub async fn reconcile_airflow( common_configuration, &metadata_database_connection_details, &validated.image, - &dereferenced.authentication_config, - &dereferenced.authorization_config, + &validated.authentication_config, + &validated.authorization_config, &mut cluster_resources, client, &rbac_sa, @@ -515,7 +510,7 @@ pub async fn reconcile_airflow( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, &validated.image, - &env_vars_from_rolegroup_config(&validated_rg_config.product_config_properties), + &env_vars_from(&validated_rg_config.overrides.env_overrides), &airflow.volume_mounts(), LOG_VOLUME_NAME, &validated_rg_config @@ -574,9 +569,9 @@ pub async fn reconcile_airflow( airflow, &validated.image, &rolegroup, - &validated_rg_config.product_config_properties, - &dereferenced.authentication_config, - &dereferenced.authorization_config, + &validated_rg_config.overrides.config_file_overrides, + &validated.authentication_config, + &validated.authorization_config, &validated_rg_config.merged_config.logging, &Container::Airflow, )?; @@ -592,9 +587,9 @@ pub async fn reconcile_airflow( &validated.image, airflow_role, &rolegroup, - &validated_rg_config.product_config_properties, - &dereferenced.authentication_config, - &dereferenced.authorization_config, + &validated_rg_config.overrides.env_overrides, + &validated.authentication_config, + &validated.authorization_config, &metadata_database_connection_details, &celery_database_connection_details, &rbac_sa, @@ -659,7 +654,7 @@ async fn build_executor_template( airflow, resolved_product_image, &rolegroup, - &HashMap::new(), + &BTreeMap::new(), authentication_config, authorization_config, &merged_executor_config.logging, @@ -709,7 +704,7 @@ fn build_rolegroup_config_map( airflow: &v1alpha2::AirflowCluster, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, - rolegroup_config: &HashMap>, + config_file_overrides: &BTreeMap, authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, logging: &Logging, @@ -732,10 +727,7 @@ fn build_rolegroup_config_map( config ); - let mut file_config = rolegroup_config - .get(&PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.to_string())) - .cloned() - .unwrap_or_default(); + let mut file_config = config_file_overrides.clone(); tracing::debug!( "Config overrides for {}: {:?}", @@ -884,7 +876,7 @@ fn build_server_rolegroup_statefulset( resolved_product_image: &ResolvedProductImage, airflow_role: &AirflowRole, rolegroup_ref: &RoleGroupRef, - rolegroup_config: &HashMap>, + env_overrides: &HashMap, authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, @@ -974,7 +966,7 @@ fn build_server_rolegroup_statefulset( env_vars::build_airflow_statefulset_envs( airflow, airflow_role, - rolegroup_config, + env_overrides, executor, authentication_config, authorization_config, diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index 6eb2a16c..f58da226 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -1,18 +1,18 @@ -use std::{ - collections::{BTreeMap, HashMap}, - str::FromStr, -}; +use std::collections::BTreeMap; -use product_config::{ProductConfigManager, types::PropertyNameKind}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ commons::product_image_selection::{self, ResolvedProductImage}, - product_config_utils::{transform_all_roles_to_config, validate_all_roles_and_groups_config}, role_utils::RoleGroupRef, }; use strum::IntoEnumIterator; -use crate::crd::{AIRFLOW_CONFIG_FILENAME, AirflowConfig, AirflowExecutor, AirflowRole, v1alpha2}; +use super::dereference::DereferencedObjects; +use crate::crd::{ + AirflowConfig, AirflowExecutor, AirflowRole, MergedOverrides, + authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, v1alpha2, +}; #[derive(Snafu, Debug)] pub enum Error { @@ -21,24 +21,8 @@ pub enum Error { source: product_image_selection::Error, }, - #[snafu(display("invalid product config"))] - InvalidProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - - #[snafu(display("failed to generate product config"))] - GenerateProductConfig { - source: stackable_operator::product_config_utils::Error, - }, - #[snafu(display("failed to resolve and merge config for role and role group"))] FailedToResolveConfig { source: crate::crd::Error }, - - #[snafu(display("could not parse Airflow role [{role}]"))] - UnidentifiedAirflowRole { - source: strum::ParseError, - role: String, - }, } /// Per-role configuration extracted during validation. @@ -49,21 +33,24 @@ pub struct ValidatedRoleConfig { pub group_listener_name: Option, } -/// Per-rolegroup configuration: the merged CRD config plus the product-config properties. +/// Per-rolegroup configuration: the merged CRD config plus overrides. #[derive(Clone, Debug)] pub struct ValidatedRoleGroupConfig { pub merged_config: AirflowConfig, - pub product_config_properties: HashMap>, + pub overrides: MergedOverrides, } -/// The validated cluster: proves that product-config validation and config merging -/// succeeded for every role and role group before any resources are created. +/// The validated cluster: proves that config merging succeeded for every role and +/// role group before any resources are created. It also carries the dereferenced +/// external references, so every downstream build step reads them from here. #[derive(Clone, Debug)] pub struct ValidatedAirflowCluster { pub image: ResolvedProductImage, pub role_groups: BTreeMap>, pub role_configs: BTreeMap, pub executor: AirflowExecutor, + pub authentication_config: AirflowClientAuthenticationDetailsResolved, + pub authorization_config: AirflowAuthorizationResolved, } pub fn validate_cluster( @@ -71,7 +58,7 @@ pub fn validate_cluster( image_base_name: &str, image_repository: &str, pkg_version: &str, - product_config_manager: &ProductConfigManager, + dereferenced: DereferencedObjects, ) -> Result { let resolved_product_image = airflow .spec @@ -79,85 +66,66 @@ pub fn validate_cluster( .resolve(image_base_name, image_repository, pkg_version) .context(ResolveProductImageSnafu)?; - let mut roles = HashMap::new(); + let mut role_groups = BTreeMap::new(); + let mut role_configs = BTreeMap::new(); // if the kubernetes executor is specified there will be no worker role as the pods // are provisioned by airflow as defined by the task (default: one pod per task) for role in AirflowRole::iter() { - if let Some(resolved_role) = airflow.get_role(&role) { - roles.insert( - role.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(AIRFLOW_CONFIG_FILENAME.into()), - ], - resolved_role.clone(), - ), - ); - } - } - - let role_config = transform_all_roles_to_config(airflow, &roles); - let validated_role_config = validate_all_roles_and_groups_config( - &resolved_product_image.product_version, - &role_config.context(GenerateProductConfigSnafu)?, - product_config_manager, - false, - false, - ) - .context(InvalidProductConfigSnafu)?; - - let mut role_groups = BTreeMap::new(); - let mut role_configs = BTreeMap::new(); - - for (role_name, rolegroup_configs) in validated_role_config.iter() { - let airflow_role = - AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { - role: role_name.to_string(), - })?; + let Some(resolved_role) = airflow.get_role(&role) else { + continue; + }; role_configs.insert( - airflow_role.clone(), + role.clone(), ValidatedRoleConfig { pdb: airflow - .role_config(&airflow_role) + .role_config(&role) .map(|rc| rc.pod_disruption_budget), - listener_class: airflow_role - .listener_class_name(airflow) - .map(|s| s.to_string()), - group_listener_name: airflow.group_listener_name(&airflow_role), + listener_class: role.listener_class_name(airflow).map(|s| s.to_string()), + group_listener_name: airflow.group_listener_name(&role), }, ); let mut group_configs = BTreeMap::new(); - for (rolegroup_name, rolegroup_config) in rolegroup_configs.iter() { + for rolegroup_name in resolved_role.role_groups.keys() { let rolegroup_ref = RoleGroupRef { cluster: stackable_operator::kube::runtime::reflector::ObjectRef::from_obj(airflow), - role: role_name.into(), + role: role.to_string(), role_group: rolegroup_name.into(), }; let merged_config = airflow - .merged_config(&airflow_role, &rolegroup_ref) + .merged_config(&role, &rolegroup_ref) + .context(FailedToResolveConfigSnafu)?; + + let overrides = airflow + .merged_overrides(&role, rolegroup_name) .context(FailedToResolveConfigSnafu)?; group_configs.insert( rolegroup_name.clone(), ValidatedRoleGroupConfig { merged_config, - product_config_properties: rolegroup_config.clone(), + overrides, }, ); } - role_groups.insert(airflow_role, group_configs); + role_groups.insert(role, group_configs); } + let DereferencedObjects { + authentication_config, + authorization_config, + } = dereferenced; + Ok(ValidatedAirflowCluster { image: resolved_product_image, role_groups, role_configs, executor: airflow.spec.executor.clone(), + authentication_config, + authorization_config, }) } diff --git a/rust/operator-binary/src/crd/authorization.rs b/rust/operator-binary/src/crd/authorization.rs index 2e15912c..0fc0726c 100644 --- a/rust/operator-binary/src/crd/authorization.rs +++ b/rust/operator-binary/src/crd/authorization.rs @@ -2,6 +2,7 @@ use stackable_operator::{client::Client, commons::opa::OpaApiVersion, shared::ti use crate::crd::{AirflowAuthorization, AirflowOpaConfig, v1alpha2}; +#[derive(Clone, Debug)] pub struct AirflowAuthorizationResolved { pub opa: Option, } @@ -24,6 +25,7 @@ impl AirflowAuthorizationResolved { } } +#[derive(Clone, Debug)] pub struct OpaConfigResolved { pub connection_string: String, pub cache_entry_time_to_live: Duration, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 521a98f5..c3a416a9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -29,7 +29,6 @@ use stackable_operator::{ kube::{CustomResource, ResourceExt}, kvp::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, - product_config_utils::{self, Configuration}, product_logging::{ self, framework::{create_vector_shutdown_file_command, remove_vector_shutdown_file_command}, @@ -1042,35 +1041,6 @@ impl AirflowConfig { } } -impl Configuration for AirflowConfigFragment { - type Configurable = v1alpha2::AirflowCluster; - - fn compute_env( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_cli( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } - - fn compute_files( - &self, - _cluster: &Self::Configurable, - _role_name: &str, - _file: &str, - ) -> Result>, product_config_utils::Error> { - Ok(BTreeMap::new()) - } -} - fn default_resources(role: &AirflowRole) -> ResourcesFragment { let (cpu, memory) = match role { AirflowRole::Worker => ( diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index b4828dbe..595eb1ec 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -3,7 +3,6 @@ use std::{ path::PathBuf, }; -use product_config::types::PropertyNameKind; use snafu::Snafu; use stackable_operator::{ commons::product_image_selection::ResolvedProductImage, @@ -79,7 +78,7 @@ pub enum Error { pub fn build_airflow_statefulset_envs( airflow: &v1alpha2::AirflowCluster, airflow_role: &AirflowRole, - rolegroup_config: &HashMap>, + env_overrides: &HashMap, executor: &AirflowExecutor, auth_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, @@ -96,9 +95,6 @@ pub fn build_airflow_statefulset_envs( env.extend(static_envs(git_sync_resources)); - // environment variables - let env_vars = rolegroup_config.get(&PropertyNameKind::Env); - add_version_specific_env_vars(airflow, airflow_role, resolved_product_image, &mut env); // N.B. this has been deprecated and replaced with AIRFLOW__API__SECRET_KEY since 3.0.2. Can be removed when 3.0.1 is no longer supported. @@ -266,17 +262,15 @@ pub fn build_airflow_statefulset_envs( } // apply overrides last of all with a fixed ordering - if let Some(env_vars) = env_vars { - for (k, v) in env_vars.iter().collect::>() { - env.insert( - k.into(), - EnvVar { - name: k.to_string(), - value: Some(v.to_string()), - ..Default::default() - }, - ); - } + for (k, v) in env_overrides.iter().collect::>() { + env.insert( + k.into(), + EnvVar { + name: k.to_string(), + value: Some(v.to_string()), + ..Default::default() + }, + ); } // Needed for the `containerdebug` process to log it's tracing information to. diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index dee5603b..469a7240 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -71,11 +71,11 @@ async fn main() -> anyhow::Result<()> { .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; } Command::Run(RunArguments { - product_config, watch_namespace, operator_environment, maintenance, common, + .. }) => { // NOTE (@NickLarsenNZ): Before stackable-telemetry was used: // - The console log level was set by `AIRFLOW_OPERATOR_LOG`, and is now `CONSOLE_LOG` (when using Tracing::pre_configured). @@ -103,11 +103,6 @@ async fn main() -> anyhow::Result<()> { .run(sigterm_watcher.handle()) .map(anyhow::Ok); - let product_config = product_config.load(&[ - "deploy/config-spec/properties.yaml", - "/etc/stackable/airflow-operator/config-spec/properties.yaml", - ])?; - let client = stackable_operator::client::initialize_operator( Some(OPERATOR_NAME.to_string()), &common.cluster_info, @@ -183,7 +178,6 @@ async fn main() -> anyhow::Result<()> { Arc::new(airflow_controller::Ctx { client: client.clone(), operator_environment, - product_config, }), ) // We can let the reporting happen in the background From 1f4e3513db9614d236feba316dace59fd7b88eec Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 11:06:22 +0200 Subject: [PATCH 03/16] docs: remove product-config CLI parameter Drops the defunct --product-config section from the commandline reference and notes the removal of product-config validation in the changelog. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 1 + .../pages/reference/commandline-parameters.adoc | 13 ------------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e46705f..c6093e94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The results backend `spec.celeryExecutors.resultBackend` is now `spec.clusterConfig.celeryResultsBackend`. The broker `spec.celeryExecutors.broker` is now `spec.clusterConfig.celeryBroker`. - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#795]). +- Removed the product-config based configuration validation. Config and environment overrides are now merged directly from the CRD, and the dereferenced objects are carried on the validated cluster. The `--product-config` CLI flag is now a no-op ([#XXX]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#801]). ### Fixed diff --git a/docs/modules/airflow/pages/reference/commandline-parameters.adoc b/docs/modules/airflow/pages/reference/commandline-parameters.adoc index 7a44b610..b41a07bf 100644 --- a/docs/modules/airflow/pages/reference/commandline-parameters.adoc +++ b/docs/modules/airflow/pages/reference/commandline-parameters.adoc @@ -2,19 +2,6 @@ This operator accepts the following command line parameters: -== product-config - -*Default value*: `/etc/stackable/airflow-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values:* false - -[source] ----- -stackable-airflow-operator run --product-config /foo/bar/properties.yaml ----- - == watch-namespace *Default value*: All namespaces From 6667f36358cac0073ad0e05d4b9badb6cbee026c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 11:33:15 +0200 Subject: [PATCH 04/16] feat: remove product-config crate dependency Vendors the Flask Python-config writer locally (flask_config_writer) and drops the airflow-operator's direct product-config dependency. The rendered webserver_config.py is unchanged (the vendored writer is byte-for-byte faithful to the crate). product-config remains a transitive dependency via stackable-operator. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 1 - Cargo.nix | 4 - Cargo.toml | 1 - rust/operator-binary/Cargo.toml | 1 - .../operator-binary/src/airflow_controller.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 2 +- .../src/flask_config_writer.rs | 325 ++++++++++++++++++ rust/operator-binary/src/main.rs | 1 + 8 files changed, 329 insertions(+), 10 deletions(-) create mode 100644 rust/operator-binary/src/flask_config_writer.rs diff --git a/Cargo.lock b/Cargo.lock index d6e8310c..64a01dc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2905,7 +2905,6 @@ dependencies = [ "fnv", "futures 0.3.32", "indoc", - "product-config", "rstest", "serde", "serde_json", diff --git a/Cargo.nix b/Cargo.nix index 5ecf9a54..3ea6cbdc 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -9575,10 +9575,6 @@ rec { name = "indoc"; packageId = "indoc"; } - { - name = "product-config"; - packageId = "product-config"; - } { name = "serde"; packageId = "serde"; diff --git a/Cargo.toml b/Cargo.toml index d04c1f1e..d686760b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ edition = "2021" repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] -product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = [ "crds", "webhook", diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 2734bb4e..92feeb34 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -9,7 +9,6 @@ repository.workspace = true publish = false [dependencies] -product-config.workspace = true stackable-operator.workspace = true anyhow.workspace = true diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 7e6112f4..c803b705 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -6,7 +6,6 @@ use std::{ }; use const_format::concatcp; -use product_config::flask_app_config_writer::{self, FlaskAppConfigWriterError}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ builder::{ @@ -93,6 +92,7 @@ use crate::{ v1alpha2, }, env_vars::{self, build_airflow_template_envs}, + flask_config_writer::{self, FlaskAppConfigWriterError}, operations::{ graceful_shutdown::{ add_airflow_graceful_shutdown_config, add_executor_graceful_shutdown_config, @@ -754,7 +754,7 @@ fn build_rolegroup_config_map( let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - flask_app_config_writer::write::( + flask_config_writer::write::( &mut config_file, config.iter(), PYTHON_IMPORTS, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c3a416a9..ea8867a7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,5 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; -use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -56,6 +55,7 @@ use crate::{ CeleryBrokerConnection, CeleryResultBackendConnection, MetadataDatabaseConnection, }, }, + flask_config_writer::{FlaskAppConfigOptions, PythonType}, util::role_service_name, }; diff --git a/rust/operator-binary/src/flask_config_writer.rs b/rust/operator-binary/src/flask_config_writer.rs new file mode 100644 index 00000000..e2639c28 --- /dev/null +++ b/rust/operator-binary/src/flask_config_writer.rs @@ -0,0 +1,325 @@ +//! Writer for Flask App configurations (Python config files). +//! +//! Vendored from `product_config::flask_app_config_writer` so airflow-operator can +//! drop the `product-config` crate dependency. Applications based on the Flask App +//! Builder (e.g. Apache Airflow) use configuration files written in Python. This +//! writer only covers top-level assignments of a few primitive types and +//! expressions — it is not a general Python code generator. +//! +//! Primitive types are escaped accordingly. Python expressions are written as-is; +//! invalid expressions produce invalid configuration files. Config overrides that do +//! not map to a known option are treated as plain expressions. + +use std::{ + io::{self, Write}, + num::ParseIntError, + str::{FromStr, ParseBoolError}, +}; + +use snafu::{ResultExt, Snafu}; + +/// Errors which can occur when using this module +// Variant names share the `Error` suffix; kept as-is from the vendored +// `product_config::flask_app_config_writer` source. +#[allow(clippy::enum_variant_names)] +#[derive(Debug, Snafu)] +pub enum FlaskAppConfigWriterError { + #[snafu(display("failed to convert '{value}' into a identifier"))] + ConvertIdentifierError { value: String }, + + #[snafu(display("failed to convert '{value}' into a boolean literal"))] + ConvertBoolLiteralError { + value: String, + source: ParseBoolError, + }, + + #[snafu(display("failed to convert '{value}' into an integer literal"))] + ConvertIntLiteralError { + value: String, + source: ParseIntError, + }, + + #[snafu(display("failed to convert '{value}' into an ASCII string literal"))] + ConvertStringLiteralError { value: String }, + + #[snafu(display("failed to convert '{value}' into a Python expression"))] + ConvertExpressionError { value: String }, + + #[snafu(display("Configuration cannot be written."))] + WriteConfigError { source: io::Error }, +} + +/// Mapping from configuration options to Python types. +pub trait FlaskAppConfigOptions { + fn python_type(&self) -> PythonType; +} + +/// All supported Python types +pub enum PythonType { + /// Python identifier + Identifier, + /// Boolean literal + BoolLiteral, + /// Integer literal + IntLiteral, + /// ASCII string literal + StringLiteral, + /// Python expression + Expression, +} + +impl PythonType { + /// Converts the given string to Python. + fn convert_to_python(&self, value: &str) -> Result { + let convert = match self { + PythonType::Identifier => PythonType::convert_to_python_identifier, + PythonType::BoolLiteral => PythonType::convert_to_python_bool_literal, + PythonType::IntLiteral => PythonType::convert_to_python_int_literal, + PythonType::StringLiteral => PythonType::convert_to_python_string_literal, + PythonType::Expression => PythonType::convert_to_python_expression, + }; + + convert(value) + } + + fn convert_to_python_identifier(value: &str) -> Result { + if value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + && value + .chars() + .next() + .filter(|c| !c.is_ascii_digit()) + .is_some() + { + Ok(value.to_string()) + } else { + ConvertIdentifierSnafu { value }.fail() + } + } + + fn convert_to_python_bool_literal(value: &str) -> Result { + value + .parse::() + .map(|b| if b { "True".into() } else { "False".into() }) + .context(ConvertBoolLiteralSnafu { value }) + } + + fn convert_to_python_int_literal(value: &str) -> Result { + value + .parse::() + .map(|i| i.to_string()) + .context(ConvertIntLiteralSnafu { value }) + } + + fn convert_to_python_string_literal(value: &str) -> Result { + if value.is_ascii() { + Ok(format!("\"{}\"", value.escape_default())) + } else { + ConvertStringLiteralSnafu { value }.fail() + } + } + + fn convert_to_python_expression(value: &str) -> Result { + if !value.trim().is_empty() { + Ok(value.to_string()) + } else { + ConvertExpressionSnafu { value }.fail() + } + } +} + +/// Writes a configuration file according to the given `FlaskAppConfigOptions` type. +pub fn write<'a, O, P, W>( + writer: &mut W, + properties: P, + imports: &[&str], +) -> Result<(), FlaskAppConfigWriterError> +where + O: FlaskAppConfigOptions + FromStr, + P: Iterator, + W: Write, +{ + for import in imports { + writeln!(writer, "{import}").context(WriteConfigSnafu)?; + } + + writeln!(writer).context(WriteConfigSnafu)?; + + for (name, value) in properties { + let variable = PythonType::Identifier.convert_to_python(name)?; + + // If an option cannot be mapped to a Python type then it is a config override and treated + // as Python expression. + let content = O::from_str(name) + .map(|option| option.python_type()) + .unwrap_or(PythonType::Expression) + .convert_to_python(value)?; + + writeln!(writer, "{variable} = {content}").context(WriteConfigSnafu)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::{ + collections::BTreeMap, + str::{FromStr, from_utf8}, + }; + + use rstest::*; + + use super::{FlaskAppConfigOptions, FlaskAppConfigWriterError, PythonType, write}; + + #[rstest] + #[case::valid_identifiers_are_converted_to_python( + PythonType::Identifier, &[ + ("_", "_"), + ("a", "a"), + ("A", "A"), + ("__", "__"), + ("_a", "_a"), + ("_A", "_A"), + ("_0", "_0"), + ("SECRET_KEY", "SECRET_KEY"), + ] + )] + #[case::valid_booleans_are_converted_to_python( + PythonType::BoolLiteral, &[ + ("False", "false"), + ("True", "true"), + ] + )] + #[case::valid_integers_are_converted_to_python( + PythonType::IntLiteral, &[ + ("-9223372036854775808", "-9223372036854775808"), + ("0", "0"), + ("9223372036854775807", "9223372036854775807"), + ] + )] + #[case::valid_strings_are_converted_to_python( + PythonType::StringLiteral, &[ + (r#""""#, ""), + (r#"" ~""#, " ~"), + (r#""\t\r\n\'\"\\""#, "\t\r\n'\"\\"), + ] + )] + #[case::valid_expressions_are_converted_to_python( + PythonType::Expression, &[ + ("os.environ[\"HOME\"]", "os.environ[\"HOME\"]"), + ] + )] + fn valid_values_are_converted_to_python( + #[case] python_type: PythonType, + #[case] values: &[(&str, &str)], + ) -> Result<(), FlaskAppConfigWriterError> { + for (expected, input) in values { + assert_eq!(*expected, python_type.convert_to_python(input)?); + } + + Ok(()) + } + + #[rstest] + #[case::invalid_identifiers_are_not_converted_to_python( + PythonType::Identifier, &[ + "", "0", "-", "\n", "_-", "_\n", + ] + )] + #[case::invalid_booleans_are_not_converted_to_python( + PythonType::BoolLiteral, &[ + "", "False", "True", "0", "1", + ] + )] + #[case::invalid_integers_are_not_converted_to_python( + PythonType::IntLiteral, &[ + "", "a", "0x10", "inf", + ] + )] + #[case::invalid_strings_are_not_converted_to_python( + PythonType::StringLiteral, &[ + "ä", "❤" + ] + )] + #[case::invalid_expressions_are_not_converted_to_python( + PythonType::Expression, &[ + "" + ] + )] + fn invalid_values_are_converted_to_python( + #[case] python_type: PythonType, + #[case] values: &[&str], + ) { + for input in values { + assert!(python_type.convert_to_python(input).is_err()); + } + } + + #[test] + fn valid_options_are_written_into_a_configuration() -> Result<(), FlaskAppConfigWriterError> { + #[allow(clippy::enum_variant_names)] + enum Options { + BoolOption, + IntOption, + StringOption, + ExpressionOption, + _UnusedOption, + } + + impl FromStr for Options { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "BOOL_OPTION" => Ok(Options::BoolOption), + "INT_OPTION" => Ok(Options::IntOption), + "STRING_OPTION" => Ok(Options::StringOption), + "EXPRESSION_OPTION" => Ok(Options::ExpressionOption), + _ => Err("unknown option"), + } + } + } + + impl FlaskAppConfigOptions for Options { + fn python_type(&self) -> PythonType { + match self { + Options::BoolOption => PythonType::BoolLiteral, + Options::IntOption => PythonType::IntLiteral, + Options::StringOption => PythonType::StringLiteral, + Options::ExpressionOption => PythonType::Expression, + Options::_UnusedOption => PythonType::Expression, + } + } + } + + let config: BTreeMap<_, _> = [ + ("BOOL_OPTION", "true"), + ("INT_OPTION", "0"), + ("STRING_OPTION", ""), + ("EXPRESSION_OPTION", "{ \"key\": \"value\" }"), + ("OVERRIDDEN_OPTION", "None"), + ] + .map(|(k, v)| (k.to_string(), v.to_string())) + .into(); + + let imports = ["import module", "from module import member"]; + + let mut config_file = Vec::new(); + write::(&mut config_file, config.iter(), &imports)?; + + assert_eq!( + r#"import module +from module import member + +BOOL_OPTION = True +EXPRESSION_OPTION = { "key": "value" } +INT_OPTION = 0 +OVERRIDDEN_OPTION = None +STRING_OPTION = "" +"#, + from_utf8(&config_file).unwrap() + ); + + Ok(()) + } +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 469a7240..add0ad9c 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -44,6 +44,7 @@ mod controller; mod controller_commons; mod crd; mod env_vars; +mod flask_config_writer; mod operations; mod product_logging; mod service; From 62c00548235a9b6b9539382e2776208ba7663000 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 12:09:21 +0200 Subject: [PATCH 05/16] refactor: move vendored Flask writer to framework/flask_app_config_writer Renames the vendored writer back to its upstream name and groups it under a `framework` module (mirroring the convention in trino-operator), signalling it as vendored code that is a candidate for a shared crate. The same writer is still used by superset-operator via the product-config crate. No behaviour change; the writer body is unchanged from the crate source. Co-Authored-By: Claude Opus 4.8 --- rust/operator-binary/src/airflow_controller.rs | 4 ++-- rust/operator-binary/src/crd/mod.rs | 2 +- rust/operator-binary/src/framework.rs | 8 ++++++++ .../flask_app_config_writer.rs} | 16 +++++++++++----- rust/operator-binary/src/main.rs | 2 +- 5 files changed, 23 insertions(+), 9 deletions(-) create mode 100644 rust/operator-binary/src/framework.rs rename rust/operator-binary/src/{flask_config_writer.rs => framework/flask_app_config_writer.rs} (92%) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index c803b705..8ea806fb 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -92,7 +92,7 @@ use crate::{ v1alpha2, }, env_vars::{self, build_airflow_template_envs}, - flask_config_writer::{self, FlaskAppConfigWriterError}, + framework::flask_app_config_writer::{self, FlaskAppConfigWriterError}, operations::{ graceful_shutdown::{ add_airflow_graceful_shutdown_config, add_executor_graceful_shutdown_config, @@ -754,7 +754,7 @@ fn build_rolegroup_config_map( let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - flask_config_writer::write::( + flask_app_config_writer::write::( &mut config_file, config.iter(), PYTHON_IMPORTS, diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index ea8867a7..662e4826 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -55,7 +55,7 @@ use crate::{ CeleryBrokerConnection, CeleryResultBackendConnection, MetadataDatabaseConnection, }, }, - flask_config_writer::{FlaskAppConfigOptions, PythonType}, + framework::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}, util::role_service_name, }; diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs new file mode 100644 index 00000000..bf913815 --- /dev/null +++ b/rust/operator-binary/src/framework.rs @@ -0,0 +1,8 @@ +//! Local helpers vendored from upstream crates, grouped here as candidates for a +//! future shared home (e.g. operator-rs) once one exists. +//! +//! Mirrors the `framework` module convention used by other Stackable operators +//! (e.g. trino-operator) for code that is intentionally duplicated until it can +//! be upstreamed. + +pub mod flask_app_config_writer; diff --git a/rust/operator-binary/src/flask_config_writer.rs b/rust/operator-binary/src/framework/flask_app_config_writer.rs similarity index 92% rename from rust/operator-binary/src/flask_config_writer.rs rename to rust/operator-binary/src/framework/flask_app_config_writer.rs index e2639c28..2af7efe0 100644 --- a/rust/operator-binary/src/flask_config_writer.rs +++ b/rust/operator-binary/src/framework/flask_app_config_writer.rs @@ -1,14 +1,20 @@ //! Writer for Flask App configurations (Python config files). //! -//! Vendored from `product_config::flask_app_config_writer` so airflow-operator can -//! drop the `product-config` crate dependency. Applications based on the Flask App -//! Builder (e.g. Apache Airflow) use configuration files written in Python. This -//! writer only covers top-level assignments of a few primitive types and -//! expressions — it is not a general Python code generator. +//! Vendored verbatim from `product_config::flask_app_config_writer` so +//! airflow-operator can drop the `product-config` crate dependency. Applications +//! based on the Flask App Builder (e.g. Apache Airflow, Apache Superset) use +//! configuration files written in Python. This writer only covers top-level +//! assignments of a few primitive types and expressions — it is not a general +//! Python code generator. //! //! Primitive types are escaped accordingly. Python expressions are written as-is; //! invalid expressions produce invalid configuration files. Config overrides that do //! not map to a known option are treated as plain expressions. +//! +// TODO: This is vendored, not airflow-specific. superset-operator still depends on +// `product_config::flask_app_config_writer`; this writer is a candidate for a shared +// crate (e.g. operator-rs) so both operators can drop the product-config crate. +// Until such a home exists it is duplicated here, kept identical to the upstream source. use std::{ io::{self, Write}, diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index add0ad9c..0caa3796 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -44,7 +44,7 @@ mod controller; mod controller_commons; mod crd; mod env_vars; -mod flask_config_writer; +mod framework; mod operations; mod product_logging; mod service; From 4885d36f0d8406a44bc77e78dd7c026519d986fd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 13:37:12 +0200 Subject: [PATCH 06/16] replace usage of product_config_utils::env_vars_from with inline calls, as done with trino --- .../operator-binary/src/airflow_controller.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 8ea806fb..81dcf8f8 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -42,7 +42,7 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, PersistentVolumeClaim, PodTemplateSpec, Probe, ServiceAccount, + ConfigMap, EnvVar, PersistentVolumeClaim, PodTemplateSpec, Probe, ServiceAccount, TCPSocketAction, }, }, @@ -56,7 +56,6 @@ use stackable_operator::{ }, kvp::{Annotation, Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, - product_config_utils::env_vars_from, product_logging::{ self, framework::LoggingError, @@ -510,7 +509,16 @@ pub async fn reconcile_airflow( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, &validated.image, - &env_vars_from(&validated_rg_config.overrides.env_overrides), + &validated_rg_config + .overrides + .env_overrides + .iter() + .map(|(k, v)| EnvVar { + name: k.clone(), + value: Some(v.clone()), + ..EnvVar::default() + }) + .collect::>(), &airflow.volume_mounts(), LOG_VOLUME_NAME, &validated_rg_config @@ -670,7 +678,15 @@ async fn build_executor_template( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, resolved_product_image, - &env_vars_from(&common_config.env_overrides), + &common_config + .env_overrides + .iter() + .map(|(k, v)| EnvVar { + name: k.clone(), + value: Some(v.clone()), + ..EnvVar::default() + }) + .collect::>(), &airflow.volume_mounts(), LOG_VOLUME_NAME, &merged_executor_config From 2bd1f79364406f90da4676491581a3f6537ccd78 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 13:41:57 +0200 Subject: [PATCH 07/16] changelog update --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6093e94..15fab1eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,8 +23,10 @@ The results backend `spec.celeryExecutors.resultBackend` is now `spec.clusterConfig.celeryResultsBackend`. The broker `spec.celeryExecutors.broker` is now `spec.clusterConfig.celeryBroker`. - Internal operator refactoring: introduce dereference() and validate() steps in the reconciler ([#795]). -- Removed the product-config based configuration validation. Config and environment overrides are now merged directly from the CRD, and the dereferenced objects are carried on the validated cluster. The `--product-config` CLI flag is now a no-op ([#XXX]). - test: Bump vector-aggregator to 0.55.0, replace /graphql call with gRPC call ([#801]). +- Removed the product-config based configuration validation. + Config and environment overrides are now merged directly from the CRD, and dereferenced objects are added to the validated cluster. + The `--product-config` CLI flag is now a no-op ([#804]). ### Fixed @@ -44,6 +46,7 @@ [#795]: https://github.com/stackabletech/airflow-operator/pull/795 [#800]: https://github.com/stackabletech/airflow-operator/pull/800 [#801]: https://github.com/stackabletech/airflow-operator/pull/801 +[#804]: https://github.com/stackabletech/airflow-operator/pull/804 ## [26.3.0] - 2026-03-16 From 18f138fbb5452ef1ded601813e4e5461caee62bd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 15:10:50 +0200 Subject: [PATCH 08/16] refactor: move vendored writer to config/writer, drop framework module Relocates the vendored Flask config writer from framework/flask_app_config_writer to config/writer, matching hdfs-operator's config/writer.rs convention (no operator keeps a vendored writer under framework/, which trino reserves for v2 upstream mirrors). Converts config.rs into a config/ module. No behaviour change. Co-Authored-By: Claude Opus 4.8 --- rust/operator-binary/src/airflow_controller.rs | 18 ++++++++---------- .../src/{config.rs => config/mod.rs} | 2 ++ .../writer.rs} | 0 rust/operator-binary/src/crd/mod.rs | 2 +- rust/operator-binary/src/framework.rs | 8 -------- rust/operator-binary/src/main.rs | 1 - 6 files changed, 11 insertions(+), 20 deletions(-) rename rust/operator-binary/src/{config.rs => config/mod.rs} (99%) rename rust/operator-binary/src/{framework/flask_app_config_writer.rs => config/writer.rs} (100%) delete mode 100644 rust/operator-binary/src/framework.rs diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 81dcf8f8..655b665f 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -72,7 +72,10 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - config::{self, PYTHON_IMPORTS}, + config::{ + self, PYTHON_IMPORTS, + writer::{self, FlaskAppConfigWriterError}, + }, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ self, AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, @@ -91,7 +94,6 @@ use crate::{ v1alpha2, }, env_vars::{self, build_airflow_template_envs}, - framework::flask_app_config_writer::{self, FlaskAppConfigWriterError}, operations::{ graceful_shutdown::{ add_airflow_graceful_shutdown_config, add_executor_graceful_shutdown_config, @@ -770,14 +772,10 @@ fn build_rolegroup_config_map( let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - flask_app_config_writer::write::( - &mut config_file, - config.iter(), - PYTHON_IMPORTS, - ) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; + writer::write::(&mut config_file, config.iter(), PYTHON_IMPORTS) + .with_context(|_| BuildRoleGroupConfigFileSnafu { + rolegroup: rolegroup.clone(), + })?; if let Some(footer) = temp_file_footer { writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config/mod.rs similarity index 99% rename from rust/operator-binary/src/config.rs rename to rust/operator-binary/src/config/mod.rs index a3cd15ab..3792f4ba 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,3 +1,5 @@ +pub mod writer; + use std::collections::BTreeMap; use indoc::formatdoc; diff --git a/rust/operator-binary/src/framework/flask_app_config_writer.rs b/rust/operator-binary/src/config/writer.rs similarity index 100% rename from rust/operator-binary/src/framework/flask_app_config_writer.rs rename to rust/operator-binary/src/config/writer.rs diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 662e4826..919a879c 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -45,6 +45,7 @@ use stackable_operator::{ use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ + config::writer::{FlaskAppConfigOptions, PythonType}, crd::{ affinity::{get_affinity, get_executor_affinity}, authentication::{ @@ -55,7 +56,6 @@ use crate::{ CeleryBrokerConnection, CeleryResultBackendConnection, MetadataDatabaseConnection, }, }, - framework::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}, util::role_service_name, }; diff --git a/rust/operator-binary/src/framework.rs b/rust/operator-binary/src/framework.rs deleted file mode 100644 index bf913815..00000000 --- a/rust/operator-binary/src/framework.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Local helpers vendored from upstream crates, grouped here as candidates for a -//! future shared home (e.g. operator-rs) once one exists. -//! -//! Mirrors the `framework` module convention used by other Stackable operators -//! (e.g. trino-operator) for code that is intentionally duplicated until it can -//! be upstreamed. - -pub mod flask_app_config_writer; diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 0caa3796..469a7240 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -44,7 +44,6 @@ mod controller; mod controller_commons; mod crd; mod env_vars; -mod framework; mod operations; mod product_logging; mod service; From 9ab5f01d274a0dc30ab86134f97349df12485d0a Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 15:23:28 +0200 Subject: [PATCH 09/16] refactor: extract webserver_config.py builder into config/webserver_config Moves the webserver_config.py rendering (defaults + config overrides + the FILE_HEADER/FILE_FOOTER python blocks + the Flask writer call) out of build_rolegroup_config_map into a dedicated config::webserver_config::build(). The header/footer key constants and the related error variants move with it. Drops three debug! traces of intermediate config maps that no longer have a call site after the extraction; the rendered webserver_config.py is unchanged. Co-Authored-By: Claude Opus 4.8 --- .../operator-binary/src/airflow_controller.rs | 89 +++---------------- rust/operator-binary/src/config/mod.rs | 1 + .../src/config/webserver_config.rs | 76 ++++++++++++++++ 3 files changed, 91 insertions(+), 75 deletions(-) create mode 100644 rust/operator-binary/src/config/webserver_config.rs diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 655b665f..b7d2ee49 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,7 +1,6 @@ //! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`] use std::{ collections::{BTreeMap, BTreeSet, HashMap}, - io::Write, sync::Arc, }; @@ -72,17 +71,14 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - config::{ - self, PYTHON_IMPORTS, - writer::{self, FlaskAppConfigWriterError}, - }, + config, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ self, AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, - AirflowConfigOptions, AirflowExecutor, AirflowExecutorCommonConfiguration, AirflowRole, - CONFIG_PATH, Container, ExecutorConfig, HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, - LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, - STACKABLE_LOG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + AirflowExecutor, AirflowExecutorCommonConfiguration, AirflowRole, CONFIG_PATH, Container, + ExecutorConfig, HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, + LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, + TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -112,9 +108,6 @@ pub const CONTAINER_IMAGE_BASE_NAME: &str = "airflow"; pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); -const CONFIG_OVERRIDE_FILE_HEADER_KEY: &str = "FILE_HEADER"; -const CONFIG_OVERRIDE_FILE_FOOTER_KEY: &str = "FILE_FOOTER"; - pub struct Ctx { pub client: stackable_operator::client::Client, pub operator_environment: OperatorEnvironmentOptions, @@ -164,9 +157,9 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to build config file for {rolegroup}"))] - BuildRoleGroupConfigFile { - source: FlaskAppConfigWriterError, + #[snafu(display("failed to build webserver config for {rolegroup}"))] + BuildWebserverConfig { + source: config::webserver_config::Error, rolegroup: RoleGroupRef, }, @@ -257,14 +250,6 @@ pub enum Error { source: stackable_operator::builder::meta::Error, }, - #[snafu(display("failed to construct config"))] - ConstructConfig { source: config::Error }, - - #[snafu(display( - "failed to write to String (Vec to be precise) containing Airflow config" - ))] - WriteToConfigFileString { source: std::io::Error }, - #[snafu(display("failed to configure logging"))] ConfigureLogging { source: LoggingError }, @@ -728,58 +713,15 @@ fn build_rolegroup_config_map( logging: &Logging, container: &Container, ) -> Result { - let mut config: BTreeMap = BTreeMap::new(); - - // this will call default values from AirflowClientAuthenticationDetails - config::add_airflow_config( - &mut config, + let config_file = config::webserver_config::build( authentication_config, authorization_config, &resolved_product_image.product_version, + config_file_overrides, ) - .context(ConstructConfigSnafu)?; - - tracing::debug!( - "Default config for {}: {:?}", - rolegroup.object_name(), - config - ); - - let mut file_config = config_file_overrides.clone(); - - tracing::debug!( - "Config overrides for {}: {:?}", - rolegroup.object_name(), - file_config - ); - - // now add any overrides, replacing any defaults - config.append(&mut file_config); - - tracing::debug!( - "Merged config for {}: {:?}", - rolegroup.object_name(), - config - ); - - let mut config_file = Vec::new(); - - // By removing the keys from `config_properties`, we avoid pasting the Python code into a Python variable as well - // (which would be bad) - if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { - writeln!(config_file, "{}", header).context(WriteToConfigFileStringSnafu)?; - } - - let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - - writer::write::(&mut config_file, config.iter(), PYTHON_IMPORTS) - .with_context(|_| BuildRoleGroupConfigFileSnafu { - rolegroup: rolegroup.clone(), - })?; - - if let Some(footer) = temp_file_footer { - writeln!(config_file, "{}", footer).context(WriteToConfigFileStringSnafu)?; - } + .with_context(|_| BuildWebserverConfigSnafu { + rolegroup: rolegroup.clone(), + })?; let mut cm_builder = ConfigMapBuilder::new(); @@ -800,10 +742,7 @@ fn build_rolegroup_config_map( .context(ObjectMetaSnafu)? .build(), ) - .add_data( - AIRFLOW_CONFIG_FILENAME, - String::from_utf8(config_file).unwrap(), - ); + .add_data(AIRFLOW_CONFIG_FILENAME, config_file); extend_config_map_with_log_config( rolegroup, diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 3792f4ba..49899f42 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,3 +1,4 @@ +pub mod webserver_config; pub mod writer; use std::collections::BTreeMap; diff --git a/rust/operator-binary/src/config/webserver_config.rs b/rust/operator-binary/src/config/webserver_config.rs new file mode 100644 index 00000000..17311f0c --- /dev/null +++ b/rust/operator-binary/src/config/webserver_config.rs @@ -0,0 +1,76 @@ +//! Builds the `webserver_config.py` Flask configuration file from the resolved +//! authentication/authorization config plus user-provided config overrides. + +use std::{collections::BTreeMap, io::Write}; + +use snafu::{ResultExt, Snafu}; + +use super::{PYTHON_IMPORTS, add_airflow_config, writer}; +use crate::crd::{ + AirflowConfigOptions, authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, +}; + +/// Marks arbitrary Python code to prepend verbatim to the generated file. +const CONFIG_OVERRIDE_FILE_HEADER_KEY: &str = "FILE_HEADER"; +/// Marks arbitrary Python code to append verbatim to the generated file. +const CONFIG_OVERRIDE_FILE_FOOTER_KEY: &str = "FILE_FOOTER"; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to construct the webserver config"))] + ConstructConfig { source: super::Error }, + + #[snafu(display("failed to write the webserver config file"))] + WriteConfigFile { + source: writer::FlaskAppConfigWriterError, + }, + + #[snafu(display("failed to write the header/footer to the webserver config file"))] + WriteHeaderFooter { source: std::io::Error }, +} + +/// Renders the `webserver_config.py` contents: operator defaults (derived from the +/// resolved authentication/authorization config) with the user's `config_overrides` +/// applied last, wrapped by the optional `FILE_HEADER`/`FILE_FOOTER` Python blocks. +pub fn build( + authentication_config: &AirflowClientAuthenticationDetailsResolved, + authorization_config: &AirflowAuthorizationResolved, + product_version: &str, + config_file_overrides: &BTreeMap, +) -> Result { + let mut config: BTreeMap = BTreeMap::new(); + + // this will call default values from AirflowClientAuthenticationDetails + add_airflow_config( + &mut config, + authentication_config, + authorization_config, + product_version, + ) + .context(ConstructConfigSnafu)?; + + let mut file_config = config_file_overrides.clone(); + + // now add any overrides, replacing any defaults + config.append(&mut file_config); + + let mut config_file = Vec::new(); + + // By removing the keys from `config`, we avoid pasting the Python code into a Python variable as well + // (which would be bad) + if let Some(header) = config.remove(CONFIG_OVERRIDE_FILE_HEADER_KEY) { + writeln!(config_file, "{}", header).context(WriteHeaderFooterSnafu)?; + } + + let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); + + writer::write::(&mut config_file, config.iter(), PYTHON_IMPORTS) + .context(WriteConfigFileSnafu)?; + + if let Some(footer) = temp_file_footer { + writeln!(config_file, "{}", footer).context(WriteHeaderFooterSnafu)?; + } + + Ok(String::from_utf8(config_file).expect("the Flask config writer only emits valid UTF-8")) +} From fc96e05ac7136254071cc91356aff34ff04d2762 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 15:26:17 +0200 Subject: [PATCH 10/16] use constants directly when resolving image --- .../operator-binary/src/airflow_controller.rs | 2 -- .../src/controller/validate.rs | 19 ++++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index b7d2ee49..468bded9 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -364,9 +364,7 @@ pub async fn reconcile_airflow( let validated = crate::controller::validate::validate_cluster( airflow, - CONTAINER_IMAGE_BASE_NAME, &ctx.operator_environment.image_repository, - crate::built_info::PKG_VERSION, dereferenced, ) .context(ValidateSnafu)?; diff --git a/rust/operator-binary/src/controller/validate.rs b/rust/operator-binary/src/controller/validate.rs index f58da226..c0ce093f 100644 --- a/rust/operator-binary/src/controller/validate.rs +++ b/rust/operator-binary/src/controller/validate.rs @@ -8,10 +8,13 @@ use stackable_operator::{ use strum::IntoEnumIterator; use super::dereference::DereferencedObjects; -use crate::crd::{ - AirflowConfig, AirflowExecutor, AirflowRole, MergedOverrides, - authentication::AirflowClientAuthenticationDetailsResolved, - authorization::AirflowAuthorizationResolved, v1alpha2, +use crate::{ + airflow_controller::CONTAINER_IMAGE_BASE_NAME, + crd::{ + AirflowConfig, AirflowExecutor, AirflowRole, MergedOverrides, + authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, v1alpha2, + }, }; #[derive(Snafu, Debug)] @@ -55,15 +58,17 @@ pub struct ValidatedAirflowCluster { pub fn validate_cluster( airflow: &v1alpha2::AirflowCluster, - image_base_name: &str, image_repository: &str, - pkg_version: &str, dereferenced: DereferencedObjects, ) -> Result { let resolved_product_image = airflow .spec .image - .resolve(image_base_name, image_repository, pkg_version) + .resolve( + CONTAINER_IMAGE_BASE_NAME, + image_repository, + crate::built_info::PKG_VERSION, + ) .context(ResolveProductImageSnafu)?; let mut role_groups = BTreeMap::new(); From 0fed7d5c4a609e0d459042013b79f1203e3c814e Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 15:27:50 +0200 Subject: [PATCH 11/16] remove mention of product-config from env-var docs --- .../reference/environment-variables.adoc | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/docs/modules/airflow/pages/reference/environment-variables.adoc b/docs/modules/airflow/pages/reference/environment-variables.adoc index 95824a7f..ea718826 100644 --- a/docs/modules/airflow/pages/reference/environment-variables.adoc +++ b/docs/modules/airflow/pages/reference/environment-variables.adoc @@ -33,32 +33,6 @@ docker run \ oci.stackable.tech/sdp/airflow-operator:0.0.0-dev ---- -== PRODUCT_CONFIG - -*Default value*: `/etc/stackable/airflow-operator/config-spec/properties.yaml` - -*Required*: false - -*Multiple values*: false - -[source] ----- -export PRODUCT_CONFIG=/foo/bar/properties.yaml -stackable-airflow-operator run ----- - -or via docker: - ----- -docker run \ - --name airflow-operator \ - --network host \ - --env KUBECONFIG=/home/stackable/.kube/config \ - --env PRODUCT_CONFIG=/my/product/config.yaml \ - --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \ - oci.stackable.tech/sdp/airflow-operator:0.0.0-dev ----- - == WATCH_NAMESPACE *Default value*: All namespaces From 58c9d03f907710e8877f1e72315be6bebef95e61 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 15:50:43 +0200 Subject: [PATCH 12/16] refactor: move rolegroup ConfigMap build into controller/build/config_map Extracts build_rolegroup_config_map out of airflow_controller into a dedicated controller/build/config_map module with its own error enum, matching the controller/build/config_map.rs layout in hdfs- and trino-operator. The controller now wraps it via a single BuildConfigMap error variant; the ConfigMap-only error variants move into the new module. No behaviour change. Co-Authored-By: Claude Opus 4.8 --- .../operator-binary/src/airflow_controller.rs | 110 +++-------------- .../src/controller/build/config_map.rs | 115 ++++++++++++++++++ .../src/controller/build/mod.rs | 3 + rust/operator-binary/src/controller/mod.rs | 1 + 4 files changed, 135 insertions(+), 94 deletions(-) create mode 100644 rust/operator-binary/src/controller/build/config_map.rs create mode 100644 rust/operator-binary/src/controller/build/mod.rs diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 468bded9..fea8052b 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -55,11 +55,7 @@ use stackable_operator::{ }, kvp::{Annotation, Label, LabelError, Labels, ObjectLabels}, logging::controller::ReconcilerError, - product_logging::{ - self, - framework::LoggingError, - spec::{ContainerLogConfig, Logging}, - }, + product_logging::{self, framework::LoggingError, spec::ContainerLogConfig}, role_utils::RoleGroupRef, shared::time::Duration, status::condition::{ @@ -71,14 +67,14 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - config, + controller::build::config_map, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ - self, AIRFLOW_CONFIG_FILENAME, APP_NAME, AirflowClusterStatus, AirflowConfig, - AirflowExecutor, AirflowExecutorCommonConfiguration, AirflowRole, CONFIG_PATH, Container, - ExecutorConfig, HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, - LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, - TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + self, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowExecutor, + AirflowExecutorCommonConfiguration, AirflowRole, CONFIG_PATH, Container, ExecutorConfig, + HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, + METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_LOCATION, + TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -96,7 +92,6 @@ use crate::{ }, pdb::add_pdbs, }, - product_logging::extend_config_map_with_log_config, service::{ build_rolegroup_headless_service, build_rolegroup_metrics_service, stateful_set_service_name, @@ -157,16 +152,9 @@ pub enum Error { source: stackable_operator::commons::rbac::Error, }, - #[snafu(display("failed to build webserver config for {rolegroup}"))] - BuildWebserverConfig { - source: config::webserver_config::Error, - rolegroup: RoleGroupRef, - }, - - #[snafu(display("failed to build ConfigMap for {rolegroup}"))] - BuildRoleGroupConfig { - source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + #[snafu(display("failed to build rolegroup ConfigMap"))] + BuildConfigMap { + source: crate::controller::build::config_map::Error, }, #[snafu(display("failed to resolve and merge config for role and role group"))] @@ -193,12 +181,6 @@ pub enum Error { #[snafu(display("vector agent is enabled but vector aggregator ConfigMap is missing"))] VectorAggregatorConfigMapMissing, - #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] - InvalidLoggingConfig { - source: crate::product_logging::Error, - cm_name: String, - }, - #[snafu(display("failed to update status"))] ApplyStatus { source: stackable_operator::client::Error, @@ -558,7 +540,7 @@ pub async fn reconcile_airflow( rolegroup: rolegroup.clone(), })?; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = config_map::build_rolegroup_config_map( airflow, &validated.image, &rolegroup, @@ -567,7 +549,8 @@ pub async fn reconcile_airflow( &validated.authorization_config, &validated_rg_config.merged_config.logging, &Container::Airflow, - )?; + ) + .context(BuildConfigMapSnafu)?; cluster_resources .add(client, rg_configmap) .await @@ -643,7 +626,7 @@ async fn build_executor_template( role_group: "kubernetes".into(), }; - let rg_configmap = build_rolegroup_config_map( + let rg_configmap = config_map::build_rolegroup_config_map( airflow, resolved_product_image, &rolegroup, @@ -652,7 +635,8 @@ async fn build_executor_template( authorization_config, &merged_executor_config.logging, &Container::Base, - )?; + ) + .context(BuildConfigMapSnafu)?; cluster_resources .add(client, rg_configmap) .await @@ -699,68 +683,6 @@ async fn build_executor_template( Ok(()) } -/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator -#[allow(clippy::too_many_arguments)] -fn build_rolegroup_config_map( - airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, - config_file_overrides: &BTreeMap, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, - logging: &Logging, - container: &Container, -) -> Result { - let config_file = config::webserver_config::build( - authentication_config, - authorization_config, - &resolved_product_image.product_version, - config_file_overrides, - ) - .with_context(|_| BuildWebserverConfigSnafu { - rolegroup: rolegroup.clone(), - })?; - - let mut cm_builder = ConfigMapBuilder::new(); - - cm_builder - .metadata( - ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(rolegroup.object_name()) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(&build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, - &rolegroup.role, - &rolegroup.role_group, - )) - .context(ObjectMetaSnafu)? - .build(), - ) - .add_data(AIRFLOW_CONFIG_FILENAME, config_file); - - extend_config_map_with_log_config( - rolegroup, - logging, - container, - &Container::Vector, - &mut cm_builder, - resolved_product_image, - ) - .context(InvalidLoggingConfigSnafu { - cm_name: rolegroup.object_name(), - })?; - - cm_builder - .build() - .with_context(|_| BuildRoleGroupConfigSnafu { - rolegroup: rolegroup.clone(), - }) -} - fn build_rolegroup_metadata( airflow: &v1alpha2::AirflowCluster, resolved_product_image: &&ResolvedProductImage, diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs new file mode 100644 index 00000000..d6351cfb --- /dev/null +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -0,0 +1,115 @@ +//! Builds the rolegroup [`ConfigMap`]: the rendered `webserver_config.py` plus the +//! logging/vector configuration. + +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::ConfigMap, + product_logging::spec::Logging, + role_utils::RoleGroupRef, +}; + +use crate::{ + airflow_controller::AIRFLOW_CONTROLLER_NAME, + config::webserver_config, + crd::{ + AIRFLOW_CONFIG_FILENAME, Container, + authentication::AirflowClientAuthenticationDetailsResolved, + authorization::AirflowAuthorizationResolved, build_recommended_labels, v1alpha2, + }, + product_logging::extend_config_map_with_log_config, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("failed to build webserver config for {rolegroup}"))] + BuildWebserverConfig { + source: webserver_config::Error, + rolegroup: RoleGroupRef, + }, + + #[snafu(display("object is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to build object meta"))] + ObjectMeta { + source: stackable_operator::builder::meta::Error, + }, + + #[snafu(display("failed to add the logging configuration to the ConfigMap [{cm_name}]"))] + InvalidLoggingConfig { + source: crate::product_logging::Error, + cm_name: String, + }, + + #[snafu(display("failed to build ConfigMap for {rolegroup}"))] + BuildConfigMap { + source: stackable_operator::builder::configmap::Error, + rolegroup: RoleGroupRef, + }, +} + +/// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator +#[allow(clippy::too_many_arguments)] +pub fn build_rolegroup_config_map( + airflow: &v1alpha2::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, + config_file_overrides: &BTreeMap, + authentication_config: &AirflowClientAuthenticationDetailsResolved, + authorization_config: &AirflowAuthorizationResolved, + logging: &Logging, + container: &Container, +) -> Result { + let config_file = webserver_config::build( + authentication_config, + authorization_config, + &resolved_product_image.product_version, + config_file_overrides, + ) + .with_context(|_| BuildWebserverConfigSnafu { + rolegroup: rolegroup.clone(), + })?; + + let mut cm_builder = ConfigMapBuilder::new(); + + cm_builder + .metadata( + ObjectMetaBuilder::new() + .name_and_namespace(airflow) + .name(rolegroup.object_name()) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(&build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + &resolved_product_image.app_version_label_value, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(ObjectMetaSnafu)? + .build(), + ) + .add_data(AIRFLOW_CONFIG_FILENAME, config_file); + + extend_config_map_with_log_config( + rolegroup, + logging, + container, + &Container::Vector, + &mut cm_builder, + resolved_product_image, + ) + .context(InvalidLoggingConfigSnafu { + cm_name: rolegroup.object_name(), + })?; + + cm_builder.build().with_context(|_| BuildConfigMapSnafu { + rolegroup: rolegroup.clone(), + }) +} diff --git a/rust/operator-binary/src/controller/build/mod.rs b/rust/operator-binary/src/controller/build/mod.rs new file mode 100644 index 00000000..881a36f4 --- /dev/null +++ b/rust/operator-binary/src/controller/build/mod.rs @@ -0,0 +1,3 @@ +//! Builders that assemble Kubernetes resources from the validated cluster. + +pub mod config_map; diff --git a/rust/operator-binary/src/controller/mod.rs b/rust/operator-binary/src/controller/mod.rs index 1b261dfe..a1196d5d 100644 --- a/rust/operator-binary/src/controller/mod.rs +++ b/rust/operator-binary/src/controller/mod.rs @@ -1,2 +1,3 @@ +pub mod build; pub mod dereference; pub mod validate; From a66bc67e6fdeb80ab1703fa0f90617cd3ffe7e31 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 16:43:04 +0200 Subject: [PATCH 13/16] pass validated cluster instead of individual parameters --- .../operator-binary/src/airflow_controller.rs | 46 ++++++++----------- .../src/controller/build/config_map.rs | 22 ++++----- 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index fea8052b..ca1fcf65 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -67,7 +67,7 @@ use stackable_operator::{ use strum::{EnumDiscriminants, IntoStaticStr}; use crate::{ - controller::build::config_map, + controller::{build::config_map, validate::ValidatedAirflowCluster}, controller_commons::{self, CONFIG_VOLUME_NAME, LOG_CONFIG_VOLUME_NAME, LOG_VOLUME_NAME}, crd::{ self, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowExecutor, @@ -344,7 +344,7 @@ pub async fn reconcile_airflow( None }; - let validated = crate::controller::validate::validate_cluster( + let validated_cluster = crate::controller::validate::validate_cluster( airflow, &ctx.operator_environment.image_repository, dereferenced, @@ -418,15 +418,13 @@ pub async fn reconcile_airflow( // collection there will be a pod template created to be used for pod provisioning if let AirflowExecutor::KubernetesExecutors { common_configuration, - } = &validated.executor + } = &validated_cluster.executor { build_executor_template( airflow, common_configuration, &metadata_database_connection_details, - &validated.image, - &validated.authentication_config, - &validated.authorization_config, + &validated_cluster, &mut cluster_resources, client, &rbac_sa, @@ -434,10 +432,10 @@ pub async fn reconcile_airflow( .await?; } - for (airflow_role, role_group_configs) in &validated.role_groups { + for (airflow_role, role_group_configs) in &validated_cluster.role_groups { let role_name = airflow_role.to_string(); - if let Some(role_config) = validated.role_configs.get(airflow_role) { + if let Some(role_config) = validated_cluster.role_configs.get(airflow_role) { if let Some(pdb) = &role_config.pdb { add_pdbs(pdb, airflow, airflow_role, client, &mut cluster_resources) .await @@ -451,7 +449,7 @@ pub async fn reconcile_airflow( build_recommended_labels( airflow, AIRFLOW_CONTROLLER_NAME, - &validated.image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &role_name, "none", ), @@ -475,7 +473,7 @@ pub async fn reconcile_airflow( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, - &validated.image, + &validated_cluster.image, &validated_rg_config .overrides .env_overrides @@ -498,7 +496,7 @@ pub async fn reconcile_airflow( let role_group_service_recommended_labels = build_recommended_labels( airflow, AIRFLOW_CONTROLLER_NAME, - &validated.image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, ); @@ -542,11 +540,9 @@ pub async fn reconcile_airflow( let rg_configmap = config_map::build_rolegroup_config_map( airflow, - &validated.image, + &validated_cluster, &rolegroup, &validated_rg_config.overrides.config_file_overrides, - &validated.authentication_config, - &validated.authorization_config, &validated_rg_config.merged_config.logging, &Container::Airflow, ) @@ -560,17 +556,17 @@ pub async fn reconcile_airflow( let rg_statefulset = build_server_rolegroup_statefulset( airflow, - &validated.image, + &validated_cluster.image, airflow_role, &rolegroup, &validated_rg_config.overrides.env_overrides, - &validated.authentication_config, - &validated.authorization_config, + &validated_cluster.authentication_config, + &validated_cluster.authorization_config, &metadata_database_connection_details, &celery_database_connection_details, &rbac_sa, &validated_rg_config.merged_config, - &validated.executor, + &validated_cluster.executor, &git_sync_resources, )?; @@ -610,9 +606,7 @@ async fn build_executor_template( airflow: &v1alpha2::AirflowCluster, common_config: &AirflowExecutorCommonConfiguration, metadata_database_connection_details: &SqlAlchemyDatabaseConnectionDetails, - resolved_product_image: &ResolvedProductImage, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, + validated_cluster: &ValidatedAirflowCluster, cluster_resources: &mut ClusterResources<'_>, client: &stackable_operator::client::Client, rbac_sa: &stackable_operator::k8s_openapi::api::core::v1::ServiceAccount, @@ -628,11 +622,9 @@ async fn build_executor_template( let rg_configmap = config_map::build_rolegroup_config_map( airflow, - resolved_product_image, + validated_cluster, &rolegroup, &BTreeMap::new(), - authentication_config, - authorization_config, &merged_executor_config.logging, &Container::Base, ) @@ -646,7 +638,7 @@ async fn build_executor_template( let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, - resolved_product_image, + &validated_cluster.image, &common_config .env_overrides .iter() @@ -666,8 +658,8 @@ async fn build_executor_template( let worker_pod_template_config_map = build_executor_template_config_map( airflow, - resolved_product_image, - authentication_config, + &validated_cluster.image, + &validated_cluster.authentication_config, metadata_database_connection_details, &rbac_sa.name_unchecked(), &merged_executor_config, diff --git a/rust/operator-binary/src/controller/build/config_map.rs b/rust/operator-binary/src/controller/build/config_map.rs index d6351cfb..b1bc88ab 100644 --- a/rust/operator-binary/src/controller/build/config_map.rs +++ b/rust/operator-binary/src/controller/build/config_map.rs @@ -6,7 +6,6 @@ use std::collections::BTreeMap; use snafu::{ResultExt, Snafu}; use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::ConfigMap, product_logging::spec::Logging, role_utils::RoleGroupRef, @@ -15,11 +14,8 @@ use stackable_operator::{ use crate::{ airflow_controller::AIRFLOW_CONTROLLER_NAME, config::webserver_config, - crd::{ - AIRFLOW_CONFIG_FILENAME, Container, - authentication::AirflowClientAuthenticationDetailsResolved, - authorization::AirflowAuthorizationResolved, build_recommended_labels, v1alpha2, - }, + controller::validate::ValidatedAirflowCluster, + crd::{AIRFLOW_CONFIG_FILENAME, Container, build_recommended_labels, v1alpha2}, product_logging::extend_config_map_with_log_config, }; @@ -58,18 +54,16 @@ pub enum Error { #[allow(clippy::too_many_arguments)] pub fn build_rolegroup_config_map( airflow: &v1alpha2::AirflowCluster, - resolved_product_image: &ResolvedProductImage, + validated_cluster: &ValidatedAirflowCluster, rolegroup: &RoleGroupRef, config_file_overrides: &BTreeMap, - authentication_config: &AirflowClientAuthenticationDetailsResolved, - authorization_config: &AirflowAuthorizationResolved, logging: &Logging, container: &Container, ) -> Result { let config_file = webserver_config::build( - authentication_config, - authorization_config, - &resolved_product_image.product_version, + &validated_cluster.authentication_config, + &validated_cluster.authorization_config, + &validated_cluster.image.product_version, config_file_overrides, ) .with_context(|_| BuildWebserverConfigSnafu { @@ -88,7 +82,7 @@ pub fn build_rolegroup_config_map( .with_recommended_labels(&build_recommended_labels( airflow, AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label_value, + &validated_cluster.image.app_version_label_value, &rolegroup.role, &rolegroup.role_group, )) @@ -103,7 +97,7 @@ pub fn build_rolegroup_config_map( container, &Container::Vector, &mut cm_builder, - resolved_product_image, + &validated_cluster.image, ) .context(InvalidLoggingConfigSnafu { cm_name: rolegroup.object_name(), From 7a370cfe1e58f8939603fa3f808fda2ca782ca80 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 3 Jun 2026 17:22:13 +0200 Subject: [PATCH 14/16] feat: adopt v2 config_overrides; build against the smooth-operator branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patches operator-rs to the smooth-operator branch (matching trino- and hdfs-operator) and actually consumes it: AirflowConfigOverrides.webserver_config_py now uses stackable_operator::v2::config_overrides::KeyValueConfigOverrides — the Merge-capable variant trino/hdfs use — instead of the v1 type. Drops the v1 KeyValueOverridesProvider impl and the as_product_config_overrides() call; merged_overrides reads the override map directly (role <- role-group extend), so the rendered webserver_config.py is unchanged (39 tests pass). The CRD gains `nullable: true` on the webserver_config.py override values (v2 allows null to delete a key). Regenerated extra/crds.yaml, Cargo.nix, and crate-hashes.json. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 32 +++++--- Cargo.nix | 114 +++++++++++++++++++++++----- Cargo.toml | 1 + crate-hashes.json | 18 ++--- extra/crds.yaml | 22 ++++++ rust/operator-binary/src/crd/mod.rs | 30 +++----- 6 files changed, 158 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64a01dc3..3f1c1cfb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1481,6 +1481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", + "schemars", "serde", "serde_json", "thiserror 1.0.69", @@ -1525,7 +1526,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "regex", @@ -2919,7 +2920,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "const-oid", "ecdsa", @@ -2942,8 +2943,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.111.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +version = "0.111.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "base64", "clap", @@ -2979,12 +2980,13 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "url", + "uuid", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "darling", "proc-macro2", @@ -2995,7 +2997,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.1.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "jiff", "k8s-openapi", @@ -3012,7 +3014,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "axum", "clap", @@ -3036,7 +3038,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "kube", "schemars", @@ -3050,7 +3052,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "convert_case", "convert_case_extras", @@ -3068,7 +3070,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" dependencies = [ "arc-swap", "async-trait", @@ -3649,6 +3651,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.nix b/Cargo.nix index 3ea6cbdc..41b1e39f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4714,6 +4714,11 @@ rec { name = "jsonptr"; packageId = "jsonptr"; } + { + name = "schemars"; + packageId = "schemars"; + optional = true; + } { name = "serde"; packageId = "serde"; @@ -4729,6 +4734,10 @@ rec { } ]; devDependencies = [ + { + name = "schemars"; + packageId = "schemars"; + } { name = "serde_json"; packageId = "serde_json"; @@ -4740,7 +4749,7 @@ rec { "schemars" = [ "dep:schemars" ]; "utoipa" = [ "dep:utoipa" ]; }; - resolvedDefaultFeatures = [ "default" "diff" ]; + resolvedDefaultFeatures = [ "default" "diff" "schemars" ]; }; "jsonpath-rust" = rec { crateName = "jsonpath-rust"; @@ -4866,8 +4875,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "k8s_version"; @@ -9637,8 +9646,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_certs"; @@ -9736,12 +9745,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.111.0"; + version = "0.111.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_operator"; @@ -9799,6 +9808,7 @@ rec { { name = "json-patch"; packageId = "json-patch"; + features = [ "schemars" ]; } { name = "k8s-openapi"; @@ -9902,6 +9912,10 @@ rec { packageId = "url"; features = [ "serde" ]; } + { + name = "uuid"; + packageId = "uuid"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; @@ -9920,8 +9934,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -9955,8 +9969,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_shared"; @@ -10036,8 +10050,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_telemetry"; @@ -10146,8 +10160,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_versioned"; @@ -10196,8 +10210,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -10264,8 +10278,8 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "b7c8a3a5483b4d35d0abfa11f6db6c153bda8a51"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_webhook"; @@ -12289,6 +12303,66 @@ rec { }; resolvedDefaultFeatures = [ "default" ]; }; + "uuid" = rec { + crateName = "uuid"; + version = "1.23.2"; + edition = "2021"; + sha256 = "1xy942s4z0bi8p3441wvd4ry3hx6ry1c7s6fgrr38462xqybhn6j"; + authors = [ + "Ashley Mannix" + "Dylan DPC" + "Hunar Roop Kahlon" + ]; + dependencies = [ + { + name = "js-sys"; + packageId = "js-sys"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)) && (builtins.elem "atomics" targetFeatures)); + } + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + devDependencies = [ + { + name = "wasm-bindgen"; + packageId = "wasm-bindgen"; + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "atomic" = [ "dep:atomic" ]; + "borsh" = [ "dep:borsh" "dep:borsh-derive" ]; + "bytemuck" = [ "dep:bytemuck" ]; + "default" = [ "std" ]; + "fast-rng" = [ "rng" "dep:rand" ]; + "js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + "md5" = [ "dep:md-5" ]; + "rng" = [ "dep:getrandom" ]; + "rng-getrandom" = [ "rng" "dep:getrandom" "uuid-rng-internal-lib" "uuid-rng-internal-lib/getrandom" ]; + "rng-rand" = [ "rng" "dep:rand" "uuid-rng-internal-lib" "uuid-rng-internal-lib/rand" ]; + "serde" = [ "dep:serde_core" ]; + "sha1" = [ "dep:sha1_smol" ]; + "slog" = [ "dep:slog" ]; + "std" = [ "wasm-bindgen?/std" "js-sys?/std" ]; + "uuid-rng-internal-lib" = [ "dep:uuid-rng-internal-lib" ]; + "v1" = [ "atomic" ]; + "v3" = [ "md5" ]; + "v4" = [ "rng" ]; + "v5" = [ "sha1" ]; + "v6" = [ "atomic" ]; + "v7" = [ "rng" ]; + "zerocopy" = [ "dep:zerocopy" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "valuable" = rec { crateName = "valuable"; version = "0.1.1"; diff --git a/Cargo.toml b/Cargo.toml index d686760b..b11fdea9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,5 +33,6 @@ tokio = { version = "1.40", features = ["full"] } tracing = "0.1" [patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "smooth-operator" } # stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } # stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/crate-hashes.json b/crate-hashes.json index 71fbc1c3..362b463b 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,12 +1,12 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-operator@0.111.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.111.0#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#k8s-version@0.1.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/extra/crds.yaml b/extra/crds.yaml index 9ec7978a..691e3350 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -483,6 +483,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1001,6 +1002,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -1998,6 +2000,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -2516,6 +2519,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -3069,6 +3073,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -3566,6 +3571,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -4084,6 +4090,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -4575,6 +4582,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -5093,6 +5101,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -5584,6 +5593,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -6107,6 +6117,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -6673,6 +6684,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -7191,6 +7203,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -8164,6 +8177,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -8682,6 +8696,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -9235,6 +9250,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -9732,6 +9748,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -10250,6 +10267,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -10741,6 +10759,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -11259,6 +11278,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -11750,6 +11770,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. @@ -12273,6 +12294,7 @@ spec: properties: webserver_config.py: additionalProperties: + nullable: true type: string description: |- Flat key-value overrides for `*.properties`, Hadoop XML, etc. diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 919a879c..c28543e4 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -18,7 +18,6 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, - config_overrides::{KeyValueConfigOverrides, KeyValueOverridesProvider}, crd::git_sync, deep_merger::ObjectOverrides, k8s_openapi::{ @@ -40,6 +39,7 @@ use stackable_operator::{ shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, + v2::config_overrides::KeyValueConfigOverrides, versioned::versioned, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; @@ -113,19 +113,6 @@ pub struct AirflowConfigOverrides { pub webserver_config_py: Option, } -impl KeyValueOverridesProvider for AirflowConfigOverrides { - fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { - match file { - AIRFLOW_CONFIG_FILENAME => self - .webserver_config_py - .as_ref() - .map(|o| o.as_product_config_overrides()) - .unwrap_or_default(), - _ => BTreeMap::new(), - } - } -} - #[derive(Clone, Debug)] pub struct MergedOverrides { pub env_overrides: HashMap, @@ -463,18 +450,21 @@ impl v1alpha2::AirflowCluster { let role_config = role.role_config(self)?; let mut env_overrides = role_config.config.env_overrides.clone(); + // role-level webserver_config.py overrides; role-group overrides are extended on top below + // (role-group wins). Reads the v2 KeyValueConfigOverrides map directly, like hdfs/trino. let mut file_overrides = role_config .config .config_overrides - .get_key_value_overrides(AIRFLOW_CONFIG_FILENAME); + .webserver_config_py + .as_ref() + .map(|o| o.overrides.clone()) + .unwrap_or_default(); if let Some(rg) = role_config.role_groups.get(rolegroup_name) { env_overrides.extend(rg.config.env_overrides.clone()); - let rg_file = rg - .config - .config_overrides - .get_key_value_overrides(AIRFLOW_CONFIG_FILENAME); - file_overrides.extend(rg_file); + if let Some(rg_file) = rg.config.config_overrides.webserver_config_py.as_ref() { + file_overrides.extend(rg_file.overrides.clone()); + } } let config_file_overrides = file_overrides From be5c1216ab0e9b5efa0bc45397be225ceeac8ef3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 5 Jun 2026 09:24:13 +0200 Subject: [PATCH 15/16] docs: make the vendored Flask writer doc operator-agnostic Neutralises the operator-specific wording in config/writer.rs so the file can be kept byte-identical between airflow-operator and superset-operator (which vendors the same writer), making the later move to a shared crate a trivial lift. Comment-only change. Co-Authored-By: Claude Opus 4.8 --- rust/operator-binary/src/config/writer.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/operator-binary/src/config/writer.rs b/rust/operator-binary/src/config/writer.rs index 2af7efe0..7c34dec7 100644 --- a/rust/operator-binary/src/config/writer.rs +++ b/rust/operator-binary/src/config/writer.rs @@ -1,8 +1,8 @@ //! Writer for Flask App configurations (Python config files). //! -//! Vendored verbatim from `product_config::flask_app_config_writer` so -//! airflow-operator can drop the `product-config` crate dependency. Applications -//! based on the Flask App Builder (e.g. Apache Airflow, Apache Superset) use +//! Vendored verbatim from `product_config::flask_app_config_writer` so the +//! operator does not depend on the `product-config` crate. Applications based +//! on the Flask App Builder (e.g. Apache Airflow, Apache Superset) use //! configuration files written in Python. This writer only covers top-level //! assignments of a few primitive types and expressions — it is not a general //! Python code generator. @@ -11,10 +11,10 @@ //! invalid expressions produce invalid configuration files. Config overrides that do //! not map to a known option are treated as plain expressions. //! -// TODO: This is vendored, not airflow-specific. superset-operator still depends on -// `product_config::flask_app_config_writer`; this writer is a candidate for a shared -// crate (e.g. operator-rs) so both operators can drop the product-config crate. -// Until such a home exists it is duplicated here, kept identical to the upstream source. +// TODO: This file is vendored identically in airflow-operator and superset-operator; +// it is a candidate for a shared crate (e.g. operator-rs) so the duplication can be +// removed. Until such a home exists it is kept identical in both operators and to the +// upstream source. use std::{ io::{self, Write}, From 317f0291522f99fca35189ab41c59f773a03a53c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 5 Jun 2026 13:35:55 +0200 Subject: [PATCH 16/16] refactor: consume the Flask config writer from stackable-operator Replace the vendored flask writer (rust/operator-binary/src/config/writer.rs) with stackable_operator::v2::flask_config_writer, which now hosts the same code (moved there verbatim from this repo via operator-rs #1217, merged into the smooth-operator branch). The base dependency tag moves from stackable-operator-0.111.0 to 0.111.1, matching the other operators. This is required for the [patch] to apply: cargo only substitutes a patch whose package version matches the dependency, and the smooth-operator branch carries 0.111.1. No behaviour change; rendered webserver_config.py output is byte-identical by construction (same code, new home). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 2 + Cargo.lock | 116 +++--- Cargo.nix | 337 ++++++++++-------- Cargo.toml | 2 +- crate-hashes.json | 4 +- rust/operator-binary/src/config/mod.rs | 1 - .../src/config/webserver_config.rs | 13 +- rust/operator-binary/src/config/writer.rs | 331 ----------------- rust/operator-binary/src/crd/mod.rs | 6 +- 9 files changed, 277 insertions(+), 535 deletions(-) delete mode 100644 rust/operator-binary/src/config/writer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 15fab1eb..77ef92f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ - Removed the product-config based configuration validation. Config and environment overrides are now merged directly from the CRD, and dereferenced objects are added to the validated cluster. The `--product-config` CLI flag is now a no-op ([#804]). +- The Flask config-file writer for `webserver_config.py` is now provided by `stackable-operator` + (`v2::flask_config_writer`) instead of a vendored copy ([#804]). ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 3f1c1cfb..0b263faf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1526,11 +1526,11 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "regex", - "snafu 0.9.0", + "snafu 0.9.1", ] [[package]] @@ -1851,9 +1851,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "opentelemetry" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -1865,9 +1865,9 @@ dependencies = [ [[package]] name = "opentelemetry-appender-tracing" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" dependencies = [ "opentelemetry", "tracing", @@ -1877,9 +1877,9 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", "bytes", @@ -1890,9 +1890,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f69cd6acbb9af919df949cd1ec9e5e7fdc2ef15d234b6b795aaa525cc02f71f" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ "http", "opentelemetry", @@ -1904,14 +1904,14 @@ dependencies = [ "thiserror 2.0.18", "tokio", "tonic", - "tracing", + "tonic-types", ] [[package]] name = "opentelemetry-proto" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -1922,21 +1922,22 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.31.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" +checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68" [[package]] name = "opentelemetry_sdk" -version = "0.31.0" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" dependencies = [ "futures-channel", "futures-executor", "futures-util", "opentelemetry", "percent-encoding", + "portable-atomic", "rand 0.9.4", "thiserror 2.0.18", "tokio", @@ -2219,6 +2220,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.45" @@ -2358,9 +2368,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -2376,9 +2386,6 @@ dependencies = [ "log", "percent-encoding", "pin-project-lite", - "serde", - "serde_json", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -2821,11 +2828,11 @@ dependencies = [ [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ - "snafu-derive 0.9.0", + "snafu-derive 0.9.1", ] [[package]] @@ -2853,9 +2860,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -2910,7 +2917,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator", "strum", "tokio", @@ -2920,7 +2927,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "const-oid", "ecdsa", @@ -2932,7 +2939,7 @@ dependencies = [ "rsa", "sha2", "signature", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-shared", "tokio", "tokio-rustls", @@ -2944,7 +2951,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.111.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "base64", "clap", @@ -2956,6 +2963,7 @@ dependencies = [ "futures 0.3.32", "http", "indexmap", + "java-properties", "jiff", "json-patch", "k8s-openapi", @@ -2968,7 +2976,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-operator-derive", "stackable-shared", "stackable-telemetry", @@ -2981,12 +2989,13 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "xml", ] [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "darling", "proc-macro2", @@ -2996,8 +3005,8 @@ dependencies = [ [[package]] name = "stackable-shared" -version = "0.1.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.1.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "jiff", "k8s-openapi", @@ -3006,15 +3015,15 @@ dependencies = [ "semver", "serde", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "time", ] [[package]] name = "stackable-telemetry" -version = "0.6.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +version = "0.6.4" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "axum", "clap", @@ -3025,7 +3034,7 @@ dependencies = [ "opentelemetry-semantic-conventions", "opentelemetry_sdk", "pin-project", - "snafu 0.9.0", + "snafu 0.9.1", "strum", "tokio", "tower", @@ -3038,21 +3047,21 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "kube", "schemars", "serde", "serde_json", "serde_yaml", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-versioned-macros", ] [[package]] name = "stackable-versioned-macros" version = "0.10.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "convert_case", "convert_case_extras", @@ -3070,7 +3079,7 @@ dependencies = [ [[package]] name = "stackable-webhook" version = "0.9.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#a31cd2514445b251038fc4ea7abc28c57b2a6ad9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#46cd3f93a788d44d177a8794fde91fbefa3156d7" dependencies = [ "arc-swap", "async-trait", @@ -3086,7 +3095,7 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "snafu 0.9.0", + "snafu 0.9.1", "stackable-certs", "stackable-shared", "stackable-telemetry", @@ -3422,6 +3431,17 @@ dependencies = [ "tonic", ] +[[package]] +name = "tonic-types" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a875a902255423d34c1f20838ab374126db8eb41625b7947a1d54113b0b7399" +dependencies = [ + "prost", + "prost-types", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -3533,9 +3553,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.32.1" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac28f2d093c6c477eaa76b23525478f38de514fa9aeb1285738d4b97a9552fc" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", "opentelemetry", @@ -3956,9 +3976,9 @@ dependencies = [ [[package]] name = "xml" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8aa498d22c9bbaf482329839bc5620c46be275a19a812e9a22a2b07529a642a" +checksum = "636f85e5ca6488e96401b61eb7de54f4e44755c988af0f52cf90230c312a1a89" [[package]] name = "yoke" diff --git a/Cargo.nix b/Cargo.nix index 41b1e39f..f3a57fca 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4876,7 +4876,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "k8s_version"; @@ -4895,7 +4895,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } ]; features = { @@ -6076,9 +6076,9 @@ rec { }; "opentelemetry" = rec { crateName = "opentelemetry"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "18629xsj4rsyiby9aj511q6wcw6s9m09gx3ymw1yjcvix1mcsjxq"; + sha256 = "10ln14d1jgc8rvw97mblc9blzcgpg1bimim4d170b7ia4mijq55h"; dependencies = [ { name = "futures-core"; @@ -6115,24 +6115,24 @@ rec { ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" "futures" ]; + "experimental_metrics_bound_instruments" = [ "metrics" ]; "futures" = [ "futures-core" "futures-sink" "pin-project-lite" ]; "futures-core" = [ "dep:futures-core" ]; "futures-sink" = [ "dep:futures-sink" ]; "internal-logs" = [ "tracing" ]; "pin-project-lite" = [ "dep:pin-project-lite" ]; - "spec_unstable_logs_enabled" = [ "logs" ]; "testing" = [ "trace" ]; "thiserror" = [ "dep:thiserror" ]; "trace" = [ "futures" "thiserror" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "spec_unstable_logs_enabled" "thiserror" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "futures" "futures-core" "futures-sink" "internal-logs" "logs" "metrics" "pin-project-lite" "thiserror" "trace" "tracing" ]; }; "opentelemetry-appender-tracing" = rec { crateName = "opentelemetry-appender-tracing"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "1hnwizzgfhpjfnvml638yy846py8hf2gl1n3p1igbk1srb2ilspg"; + sha256 = "0dyq4myan64sl8wly02jx0gb3jjz7575mn3w8rpphz0xvkq8001c"; libName = "opentelemetry_appender_tracing"; dependencies = [ { @@ -6175,18 +6175,15 @@ rec { ]; features = { "experimental_metadata_attributes" = [ "dep:tracing-log" ]; - "experimental_use_tracing_span_context" = [ "tracing-opentelemetry" ]; "log" = [ "dep:log" ]; - "spec_unstable_logs_enabled" = [ "opentelemetry/spec_unstable_logs_enabled" ]; - "tracing-opentelemetry" = [ "dep:tracing-opentelemetry" ]; }; resolvedDefaultFeatures = [ "default" ]; }; "opentelemetry-http" = rec { crateName = "opentelemetry-http"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0pc5nw1ds8v8w0nvyall39m92v8m1xl1p3vwvxk6nkhrffdd19np"; + sha256 = "0ca3drvm4fx5nskl7yn42dimy3bg35ppzc85y1p27pz215fh30sn"; libName = "opentelemetry_http"; dependencies = [ { @@ -6222,16 +6219,16 @@ rec { "internal-logs" = [ "opentelemetry/internal-logs" ]; "reqwest" = [ "dep:reqwest" ]; "reqwest-blocking" = [ "dep:reqwest" "reqwest/blocking" ]; - "reqwest-rustls" = [ "dep:reqwest" "reqwest/rustls-tls-native-roots" ]; - "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/rustls-tls-webpki-roots" ]; + "reqwest-rustls" = [ "dep:reqwest" "reqwest/default-tls" ]; + "reqwest-rustls-webpki-roots" = [ "dep:reqwest" "reqwest/default-tls" "reqwest/webpki-roots" ]; }; - resolvedDefaultFeatures = [ "internal-logs" "reqwest" "reqwest-blocking" ]; + resolvedDefaultFeatures = [ "reqwest" "reqwest-blocking" ]; }; "opentelemetry-otlp" = rec { crateName = "opentelemetry-otlp"; - version = "0.31.1"; + version = "0.32.0"; edition = "2021"; - sha256 = "07zp0b62b9dajnvvcd6j2ppw5zg7wp4ixka9z6fr3bxrrdmcss8z"; + sha256 = "0d9cys2flpidfxbr6h1103hjc633cax47ihnqgbj0xnicscr4rlr"; libName = "opentelemetry_otlp"; dependencies = [ { @@ -6292,10 +6289,9 @@ rec { usesDefaultFeatures = false; } { - name = "tracing"; - packageId = "tracing"; + name = "tonic-types"; + packageId = "tonic-types"; optional = true; - usesDefaultFeatures = false; } ]; devDependencies = [ @@ -6320,16 +6316,19 @@ rec { ]; features = { "default" = [ "http-proto" "reqwest-blocking-client" "trace" "metrics" "logs" "internal-logs" ]; + "experimental-grpc-retry" = [ "grpc-tonic" "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" ]; + "experimental-http-retry" = [ "opentelemetry_sdk/experimental_async_runtime" "opentelemetry_sdk/rt-tokio" "tokio" "httpdate" ]; "flate2" = [ "dep:flate2" ]; - "grpc-tonic" = [ "tonic" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; + "grpc-tonic" = [ "tonic" "tonic-types" "prost" "http" "tokio" "opentelemetry-proto/gen-tonic" ]; "gzip-http" = [ "flate2" ]; "gzip-tonic" = [ "tonic/gzip" ]; "http" = [ "dep:http" ]; "http-json" = [ "serde_json" "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "opentelemetry-proto/with-serde" "http" "trace" "metrics" ]; "http-proto" = [ "prost" "opentelemetry-http" "opentelemetry-proto/gen-tonic-messages" "http" "trace" "metrics" ]; + "httpdate" = [ "dep:httpdate" ]; "hyper-client" = [ "opentelemetry-http/hyper" ]; "integration-testing" = [ "tonic" "prost" "tokio/full" "trace" "logs" ]; - "internal-logs" = [ "tracing" "opentelemetry_sdk/internal-logs" "opentelemetry-http/internal-logs" ]; + "internal-logs" = [ "opentelemetry_sdk/internal-logs" "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" "opentelemetry-proto/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" "opentelemetry-proto/metrics" ]; "opentelemetry-http" = [ "dep:opentelemetry-http" ]; @@ -6342,27 +6341,27 @@ rec { "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; "serialize" = [ "serde" "serde_json" ]; - "tls" = [ "tonic/tls-ring" ]; + "tls" = [ "tls-ring" ]; "tls-aws-lc" = [ "tonic/tls-aws-lc" ]; "tls-provider-agnostic" = [ "tonic/_tls-any" ]; "tls-ring" = [ "tonic/tls-ring" ]; - "tls-roots" = [ "tls" "tonic/tls-native-roots" ]; - "tls-webpki-roots" = [ "tls" "tonic/tls-webpki-roots" ]; + "tls-roots" = [ "tonic/tls-native-roots" ]; + "tls-webpki-roots" = [ "tonic/tls-webpki-roots" ]; "tokio" = [ "dep:tokio" ]; "tonic" = [ "dep:tonic" ]; + "tonic-types" = [ "dep:tonic-types" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" "opentelemetry-proto/trace" ]; - "tracing" = [ "dep:tracing" ]; "zstd" = [ "dep:zstd" ]; "zstd-http" = [ "zstd" ]; "zstd-tonic" = [ "tonic/zstd" ]; }; - resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "trace" "tracing" ]; + resolvedDefaultFeatures = [ "default" "grpc-tonic" "gzip-tonic" "http" "http-proto" "internal-logs" "logs" "metrics" "opentelemetry-http" "prost" "reqwest" "reqwest-blocking-client" "tokio" "tonic" "tonic-types" "trace" ]; }; "opentelemetry-proto" = rec { crateName = "opentelemetry-proto"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "03xkjsjrsm7zkkx5gascqd9bg2z20wymm06l16cyxsp5dpq5s5x7"; + sha256 = "0f5ny4rpnpq6q5q34b8k2q548rf31rpbxkwjqjwzfqxg3yx5imjn"; libName = "opentelemetry_proto"; dependencies = [ { @@ -6406,30 +6405,29 @@ rec { "const-hex" = [ "dep:const-hex" ]; "default" = [ "full" ]; "full" = [ "gen-tonic" "trace" "logs" "metrics" "zpages" "with-serde" "internal-logs" ]; - "gen-tonic" = [ "gen-tonic-messages" "tonic/channel" ]; - "gen-tonic-messages" = [ "tonic" "tonic-prost" "prost" ]; + "gen-tonic" = [ "gen-tonic-messages" "tonic" "tonic-prost" "tonic/channel" ]; + "gen-tonic-messages" = [ "prost" ]; "internal-logs" = [ "opentelemetry/internal-logs" ]; "logs" = [ "opentelemetry/logs" "opentelemetry_sdk/logs" ]; "metrics" = [ "opentelemetry/metrics" "opentelemetry_sdk/metrics" ]; "prost" = [ "dep:prost" ]; "schemars" = [ "dep:schemars" ]; "serde" = [ "dep:serde" ]; - "serde_json" = [ "dep:serde_json" ]; "testing" = [ "opentelemetry/testing" ]; "tonic" = [ "dep:tonic" ]; "tonic-prost" = [ "dep:tonic-prost" ]; "trace" = [ "opentelemetry/trace" "opentelemetry_sdk/trace" ]; "with-schemars" = [ "schemars" ]; - "with-serde" = [ "serde" "const-hex" "base64" "serde_json" ]; + "with-serde" = [ "serde" "const-hex" "base64" ]; "zpages" = [ "trace" ]; }; resolvedDefaultFeatures = [ "gen-tonic" "gen-tonic-messages" "logs" "metrics" "prost" "tonic" "tonic-prost" "trace" ]; }; "opentelemetry-semantic-conventions" = rec { crateName = "opentelemetry-semantic-conventions"; - version = "0.31.0"; + version = "0.32.0"; edition = "2021"; - sha256 = "0in8plv2l2ar7anzi7lrbll0fjfvaymkg5vc5bnvibs1w3gjjbp6"; + sha256 = "0s1x4h1cgmhkxb7i5g02la2vhkf4lg5g26cgn2s2gd1p0j5gk8kc"; libName = "opentelemetry_semantic_conventions"; features = { }; @@ -6437,9 +6435,9 @@ rec { }; "opentelemetry_sdk" = rec { crateName = "opentelemetry_sdk"; - version = "0.31.0"; + version = "0.32.1"; edition = "2021"; - sha256 = "1gbjsggdxfpjbanjvaxa3nq32vfa37i3v13dvx4gsxhrk7sy8jp1"; + sha256 = "1ycl11syranrinhgn4c2hlzhyzyvpa06ryxq5mxgzmf4387ghncv"; dependencies = [ { name = "futures-channel"; @@ -6465,6 +6463,13 @@ rec { packageId = "percent-encoding"; optional = true; } + { + name = "portable-atomic"; + packageId = "portable-atomic"; + usesDefaultFeatures = false; + target = { target, features }: (!("64" == target."has_atomic" or null)); + features = [ "fallback" ]; + } { name = "rand"; packageId = "rand 0.9.4"; @@ -6489,10 +6494,18 @@ rec { optional = true; } ]; + devDependencies = [ + { + name = "tokio"; + packageId = "tokio"; + usesDefaultFeatures = false; + features = [ "macros" "rt-multi-thread" ]; + } + ]; features = { "default" = [ "trace" "metrics" "logs" "internal-logs" ]; "experimental_logs_batch_log_processor_with_async_runtime" = [ "logs" "experimental_async_runtime" ]; - "experimental_logs_concurrent_log_processor" = [ "logs" ]; + "experimental_metrics_bound_instruments" = [ "metrics" "opentelemetry/experimental_metrics_bound_instruments" ]; "experimental_metrics_custom_reader" = [ "metrics" ]; "experimental_metrics_disable_name_validation" = [ "metrics" ]; "experimental_metrics_periodicreader_with_async_runtime" = [ "metrics" "experimental_async_runtime" ]; @@ -6509,15 +6522,14 @@ rec { "rt-tokio-current-thread" = [ "tokio/rt" "tokio/time" "tokio-stream" "experimental_async_runtime" ]; "serde" = [ "dep:serde" ]; "serde_json" = [ "dep:serde_json" ]; - "spec_unstable_logs_enabled" = [ "logs" "opentelemetry/spec_unstable_logs_enabled" ]; "spec_unstable_metrics_views" = [ "metrics" ]; - "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "rt-tokio" "rt-tokio-current-thread" "tokio/macros" "tokio/rt-multi-thread" ]; + "testing" = [ "opentelemetry/testing" "trace" "metrics" "logs" "tokio/sync" ]; "tokio" = [ "dep:tokio" ]; "tokio-stream" = [ "dep:tokio-stream" ]; "trace" = [ "opentelemetry/trace" "rand" "percent-encoding" ]; "url" = [ "dep:url" ]; }; - resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "spec_unstable_logs_enabled" "tokio" "tokio-stream" "trace" ]; + resolvedDefaultFeatures = [ "default" "experimental_async_runtime" "internal-logs" "logs" "metrics" "percent-encoding" "rand" "rt-tokio" "tokio" "tokio-stream" "trace" ]; }; "ordered-float" = rec { crateName = "ordered-float"; @@ -7026,7 +7038,7 @@ rec { "default" = [ "fallback" ]; "serde" = [ "dep:serde" ]; }; - resolvedDefaultFeatures = [ "require-cas" ]; + resolvedDefaultFeatures = [ "fallback" "require-cas" ]; }; "portable-atomic-util" = rec { crateName = "portable-atomic-util"; @@ -7294,6 +7306,34 @@ rec { ]; }; + "prost-types" = rec { + crateName = "prost-types"; + version = "0.14.3"; + edition = "2021"; + sha256 = "1mrxrciryfgi6a0vmrgyj3g27r9hdhlgwkq71cgv3icbvg5w94c9"; + libName = "prost_types"; + authors = [ + "Dan Burkert " + "Lucio Franco " + "Casper Meijn " + "Tokio Contributors " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "chrono" = [ "dep:chrono" ]; + "default" = [ "std" ]; + "std" = [ "prost/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "quote" = rec { crateName = "quote"; version = "1.0.45"; @@ -7723,9 +7763,9 @@ rec { }; "reqwest" = rec { crateName = "reqwest"; - version = "0.12.28"; + version = "0.13.4"; edition = "2021"; - sha256 = "0iqidijghgqbzl3bjg5hb4zmigwa4r612bgi0yiq0c90b6jkrpgd"; + sha256 = "1hy1plns9krbh3h1dy2sdjygsfkdcnxm6pbxdi0ya9b5vq8mi711"; authors = [ "Sean McArthur " ]; @@ -7742,7 +7782,7 @@ rec { name = "futures-channel"; packageId = "futures-channel"; optional = true; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "futures-core"; @@ -7762,62 +7802,44 @@ rec { { name = "http-body"; packageId = "http-body"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "http-body-util"; packageId = "http-body-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "hyper"; packageId = "hyper"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "client" "client-legacy" "client-proxy" "tokio" ]; } { name = "js-sys"; packageId = "js-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "log"; packageId = "log"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "percent-encoding"; packageId = "percent-encoding"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "pin-project-lite"; packageId = "pin-project-lite"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - } - { - name = "serde"; - packageId = "serde"; - } - { - name = "serde_json"; - packageId = "serde_json"; - optional = true; - } - { - name = "serde_json"; - packageId = "serde_json"; - target = { target, features }: ("wasm32" == target."arch" or null); - } - { - name = "serde_urlencoded"; - packageId = "serde_urlencoded"; + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "sync_wrapper"; @@ -7828,27 +7850,27 @@ rec { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "net" "time" ]; } { name = "tower"; packageId = "tower"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "retry" "timeout" "util" ]; } { name = "tower-http"; packageId = "tower-http"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "follow-redirect" ]; } { name = "tower-service"; packageId = "tower-service"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); } { name = "url"; @@ -7857,17 +7879,17 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "wasm-bindgen-futures"; packageId = "wasm-bindgen-futures"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); } { name = "web-sys"; packageId = "web-sys"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "AbortController" "AbortSignal" "Headers" "Request" "RequestInit" "RequestMode" "Response" "Window" "FormData" "Blob" "BlobPropertyBag" "ServiceWorkerGlobalScope" "RequestCredentials" "File" "ReadableStream" "RequestCache" ]; } ]; @@ -7876,33 +7898,27 @@ rec { name = "futures-util"; packageId = "futures-util"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "std" "alloc" ]; } { name = "hyper"; packageId = "hyper"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "server" ]; } { name = "hyper-util"; packageId = "hyper-util"; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "http1" "http2" "client" "client-legacy" "server-auto" "server-graceful" "tokio" ]; } - { - name = "serde"; - packageId = "serde"; - target = { target, features }: (!("wasm32" == target."arch" or null)); - features = [ "derive" ]; - } { name = "tokio"; packageId = "tokio"; usesDefaultFeatures = false; - target = { target, features }: (!("wasm32" == target."arch" or null)); + target = { target, features }: (!(("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null)))); features = [ "macros" "rt-multi-thread" ]; } { @@ -7914,40 +7930,37 @@ rec { { name = "wasm-bindgen"; packageId = "wasm-bindgen"; - target = { target, features }: ("wasm32" == target."arch" or null); + target = { target, features }: (("wasm32" == target."arch" or null) && (("unknown" == target."os" or null) || ("none" == target."os" or null))); features = [ "serde-serialize" ]; } ]; features = { + "__native-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "__native-tls-alpn" = [ "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; "__rustls" = [ "dep:hyper-rustls" "dep:tokio-rustls" "dep:rustls" "__tls" ]; - "__rustls-ring" = [ "hyper-rustls?/ring" "tokio-rustls?/ring" "rustls?/ring" "quinn?/ring" ]; + "__rustls-aws-lc-rs" = [ "hyper-rustls?/aws-lc-rs" "tokio-rustls?/aws-lc-rs" "rustls?/aws-lc-rs" "quinn?/rustls-aws-lc-rs" ]; "__tls" = [ "dep:rustls-pki-types" "tokio/io-util" ]; "blocking" = [ "dep:futures-channel" "futures-channel?/sink" "dep:futures-util" "futures-util?/io" "futures-util?/sink" "tokio/sync" ]; "brotli" = [ "tower-http/decompression-br" ]; "charset" = [ "dep:encoding_rs" "dep:mime" ]; "cookies" = [ "dep:cookie_crate" "dep:cookie_store" ]; "default" = [ "default-tls" "charset" "http2" "system-proxy" ]; - "default-tls" = [ "dep:hyper-tls" "dep:native-tls-crate" "__tls" "dep:tokio-native-tls" ]; + "default-tls" = [ "rustls" ]; "deflate" = [ "tower-http/decompression-deflate" ]; + "form" = [ "dep:serde" "dep:serde_urlencoded" ]; "gzip" = [ "tower-http/decompression-gzip" ]; - "h2" = [ "dep:h2" ]; "hickory-dns" = [ "dep:hickory-resolver" "dep:once_cell" ]; - "http2" = [ "h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; - "http3" = [ "rustls-tls-manual-roots" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; - "json" = [ "dep:serde_json" ]; - "macos-system-configuration" = [ "system-proxy" ]; + "http2" = [ "dep:h2" "hyper/http2" "hyper-util/http2" "hyper-rustls?/http2" ]; + "http3" = [ "rustls" "dep:h3" "dep:h3-quinn" "dep:quinn" "tokio/macros" ]; + "json" = [ "dep:serde" "dep:serde_json" ]; "multipart" = [ "dep:mime_guess" "dep:futures-util" ]; - "native-tls" = [ "default-tls" ]; - "native-tls-alpn" = [ "native-tls" "native-tls-crate?/alpn" "hyper-tls?/alpn" ]; - "native-tls-vendored" = [ "native-tls" "native-tls-crate?/vendored" ]; - "rustls-tls" = [ "rustls-tls-webpki-roots" ]; - "rustls-tls-manual-roots" = [ "rustls-tls-manual-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-manual-roots-no-provider" = [ "__rustls" ]; - "rustls-tls-native-roots" = [ "rustls-tls-native-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-native-roots-no-provider" = [ "dep:rustls-native-certs" "hyper-rustls?/native-tokio" "__rustls" ]; - "rustls-tls-no-provider" = [ "rustls-tls-manual-roots-no-provider" ]; - "rustls-tls-webpki-roots" = [ "rustls-tls-webpki-roots-no-provider" "__rustls-ring" ]; - "rustls-tls-webpki-roots-no-provider" = [ "dep:webpki-roots" "hyper-rustls?/webpki-tokio" "__rustls" ]; + "native-tls" = [ "__native-tls" "__native-tls-alpn" ]; + "native-tls-no-alpn" = [ "__native-tls" ]; + "native-tls-vendored" = [ "__native-tls" "native-tls-crate?/vendored" "__native-tls-alpn" ]; + "native-tls-vendored-no-alpn" = [ "__native-tls" "native-tls-crate?/vendored" ]; + "query" = [ "dep:serde" "dep:serde_urlencoded" ]; + "rustls" = [ "__rustls-aws-lc-rs" "dep:rustls-platform-verifier" "__rustls" ]; + "rustls-no-provider" = [ "dep:rustls-platform-verifier" "__rustls" ]; "stream" = [ "tokio/fs" "dep:futures-util" "dep:tokio-util" "dep:wasm-streams" ]; "system-proxy" = [ "hyper-util/client-proxy-system" ]; "zstd" = [ "tower-http/decompression-zstd" ]; @@ -9321,29 +9334,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "rust_1_61" "rust_1_65" "std" ]; }; - "snafu 0.9.0" = rec { + "snafu 0.9.1" = rec { crateName = "snafu"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "1ii9r99x5qcn754m624yzgb9hzvkqkrcygf0aqh0pyb9dbnvrm6i"; + sha256 = "08k5yfydxdlshivfhrdq9km8qn02r93q28gkyvazbqz2icr1586i"; authors = [ "Jake Goulding " ]; dependencies = [ { name = "snafu-derive"; - packageId = "snafu-derive 0.9.0"; + packageId = "snafu-derive 0.9.1"; } ]; features = { - "backtrace" = [ "dep:backtrace" ]; - "backtraces-impl-backtrace-crate" = [ "backtrace" ]; + "backtraces-impl-backtrace-crate" = [ "dep:backtrace" ]; "default" = [ "std" "rust_1_81" ]; - "futures" = [ "futures-core-crate" "pin-project" ]; - "futures-core-crate" = [ "dep:futures-core-crate" ]; - "futures-crate" = [ "dep:futures-crate" ]; - "internal-dev-dependencies" = [ "futures-crate" ]; - "pin-project" = [ "dep:pin-project" ]; + "futures" = [ "dep:futures-core" "dep:pin-project" ]; + "internal-dev-dependencies" = [ "dep:futures" ]; "std" = [ "alloc" ]; "unstable-provider-api" = [ "snafu-derive/unstable-provider-api" ]; }; @@ -9411,11 +9420,11 @@ rec { }; resolvedDefaultFeatures = [ "rust_1_61" ]; }; - "snafu-derive 0.9.0" = rec { + "snafu-derive 0.9.1" = rec { crateName = "snafu-derive"; - version = "0.9.0"; + version = "0.9.1"; edition = "2018"; - sha256 = "0h0x61kyj4fvilcr2nj02l85shw1ika64vq9brf2gyna662ln9al"; + sha256 = "1nkfi7bis72pz3w7vb64m79w49qsv20sbf19jkd471vbhr83q42z"; procMacro = true; libName = "snafu_derive"; authors = [ @@ -9441,7 +9450,7 @@ rec { name = "syn"; packageId = "syn 2.0.117"; usesDefaultFeatures = false; - features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" ]; + features = [ "clone-impls" "derive" "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } ]; features = { @@ -9599,7 +9608,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator"; @@ -9647,7 +9656,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_certs"; @@ -9706,7 +9715,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-shared"; @@ -9750,7 +9759,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_operator"; @@ -9801,6 +9810,10 @@ rec { name = "indexmap"; packageId = "indexmap"; } + { + name = "java-properties"; + packageId = "java-properties"; + } { name = "jiff"; packageId = "jiff"; @@ -9858,7 +9871,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-operator-derive"; @@ -9916,12 +9929,17 @@ rec { name = "uuid"; packageId = "uuid"; } + { + name = "xml"; + packageId = "xml"; + } ]; features = { "certs" = [ "dep:stackable-certs" ]; + "client-feature-gates" = [ "dep:winnow" ]; "crds" = [ "dep:stackable-versioned" ]; "default" = [ "crds" ]; - "full" = [ "crds" "certs" "time" "webhook" "kube-ws" ]; + "full" = [ "client-feature-gates" "crds" "certs" "time" "webhook" "kube-ws" ]; "kube-ws" = [ "kube/ws" ]; "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; @@ -9935,7 +9953,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -9965,12 +9983,12 @@ rec { }; "stackable-shared" = rec { crateName = "stackable-shared"; - version = "0.1.0"; + version = "0.1.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_shared"; @@ -10015,7 +10033,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10046,12 +10064,12 @@ rec { }; "stackable-telemetry" = rec { crateName = "stackable-telemetry"; - version = "0.6.3"; + version = "0.6.4"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_telemetry"; @@ -10095,7 +10113,7 @@ rec { { name = "opentelemetry_sdk"; packageId = "opentelemetry_sdk"; - features = [ "rt-tokio" "logs" "rt-tokio" "spec_unstable_logs_enabled" ]; + features = [ "rt-tokio" "logs" "rt-tokio" ]; } { name = "pin-project"; @@ -10103,7 +10121,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "strum"; @@ -10161,7 +10179,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_versioned"; @@ -10195,7 +10213,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-versioned-macros"; @@ -10211,7 +10229,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; procMacro = true; @@ -10279,7 +10297,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "a31cd2514445b251038fc4ea7abc28c57b2a6ad9"; + rev = "46cd3f93a788d44d177a8794fde91fbefa3156d7"; sha256 = "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc"; }; libName = "stackable_webhook"; @@ -10352,7 +10370,7 @@ rec { } { name = "snafu"; - packageId = "snafu 0.9.0"; + packageId = "snafu 0.9.1"; } { name = "stackable-certs"; @@ -11443,6 +11461,33 @@ rec { } ]; + }; + "tonic-types" = rec { + crateName = "tonic-types"; + version = "0.14.5"; + edition = "2021"; + sha256 = "16bk1cxi2m0xgaabf98nnj7dn9j16ymkh27jq4s3shjm4a85m1ra"; + libName = "tonic_types"; + authors = [ + "Lucio Franco " + "Rafael Lemos " + ]; + dependencies = [ + { + name = "prost"; + packageId = "prost"; + } + { + name = "prost-types"; + packageId = "prost-types"; + } + { + name = "tonic"; + packageId = "tonic"; + usesDefaultFeatures = false; + } + ]; + }; "tower" = rec { crateName = "tower"; @@ -11899,9 +11944,9 @@ rec { }; "tracing-opentelemetry" = rec { crateName = "tracing-opentelemetry"; - version = "0.32.1"; + version = "0.33.0"; edition = "2021"; - sha256 = "1z2jjmxbkm1qawlb3bm99x8xwf4g8wjkbcknm9z4fv1w14nqzhhs"; + sha256 = "09nvxy5m7nxmifz4b6szdcyczapp2jcgxcac0jw4ax8klz5n9g5d"; libName = "tracing_opentelemetry"; dependencies = [ { @@ -14021,9 +14066,9 @@ rec { }; "xml" = rec { crateName = "xml"; - version = "1.2.1"; + version = "1.3.0"; edition = "2021"; - sha256 = "0ak4k990faralbli5a0rb8kvwihccb2rp0r94d4azfy94a6lkamq"; + sha256 = "128s58qhq8whrx90zbw8r5algr7lakgbf7mn05jfk234rbjqavv3"; authors = [ "Vladimir Matveev " "Kornel (https://github.com/kornelski)" diff --git a/Cargo.toml b/Cargo.toml index b11fdea9..89d4bbc9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ repository = "https://github.com/stackabletech/airflow-operator" stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = [ "crds", "webhook", -], tag = "stackable-operator-0.111.0" } +], tag = "stackable-operator-0.111.1" } anyhow = "1.0" built = { version = "0.8", features = ["chrono", "git2"] } diff --git a/crate-hashes.json b/crate-hashes.json index 362b463b..014d9478 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -3,8 +3,8 @@ "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-certs@0.4.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator-derive@0.3.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-operator@0.111.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", - "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.3": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-shared@0.1.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", + "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-telemetry@0.6.4": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned-macros@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-versioned@0.10.0": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", "git+https://github.com/stackabletech//operator-rs.git?branch=smooth-operator#stackable-webhook@0.9.1": "14q10sppdjdf3vbcbxz12rlgm1g9l6p87nk9wr707w2a71z8vgxc", diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 49899f42..d4213c18 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -1,5 +1,4 @@ pub mod webserver_config; -pub mod writer; use std::collections::BTreeMap; diff --git a/rust/operator-binary/src/config/webserver_config.rs b/rust/operator-binary/src/config/webserver_config.rs index 17311f0c..914e2a50 100644 --- a/rust/operator-binary/src/config/webserver_config.rs +++ b/rust/operator-binary/src/config/webserver_config.rs @@ -4,8 +4,9 @@ use std::{collections::BTreeMap, io::Write}; use snafu::{ResultExt, Snafu}; +use stackable_operator::v2::flask_config_writer; -use super::{PYTHON_IMPORTS, add_airflow_config, writer}; +use super::{PYTHON_IMPORTS, add_airflow_config}; use crate::crd::{ AirflowConfigOptions, authentication::AirflowClientAuthenticationDetailsResolved, authorization::AirflowAuthorizationResolved, @@ -23,7 +24,7 @@ pub enum Error { #[snafu(display("failed to write the webserver config file"))] WriteConfigFile { - source: writer::FlaskAppConfigWriterError, + source: flask_config_writer::FlaskAppConfigWriterError, }, #[snafu(display("failed to write the header/footer to the webserver config file"))] @@ -65,8 +66,12 @@ pub fn build( let temp_file_footer: Option = config.remove(CONFIG_OVERRIDE_FILE_FOOTER_KEY); - writer::write::(&mut config_file, config.iter(), PYTHON_IMPORTS) - .context(WriteConfigFileSnafu)?; + flask_config_writer::write::( + &mut config_file, + config.iter(), + PYTHON_IMPORTS, + ) + .context(WriteConfigFileSnafu)?; if let Some(footer) = temp_file_footer { writeln!(config_file, "{}", footer).context(WriteHeaderFooterSnafu)?; diff --git a/rust/operator-binary/src/config/writer.rs b/rust/operator-binary/src/config/writer.rs deleted file mode 100644 index 7c34dec7..00000000 --- a/rust/operator-binary/src/config/writer.rs +++ /dev/null @@ -1,331 +0,0 @@ -//! Writer for Flask App configurations (Python config files). -//! -//! Vendored verbatim from `product_config::flask_app_config_writer` so the -//! operator does not depend on the `product-config` crate. Applications based -//! on the Flask App Builder (e.g. Apache Airflow, Apache Superset) use -//! configuration files written in Python. This writer only covers top-level -//! assignments of a few primitive types and expressions — it is not a general -//! Python code generator. -//! -//! Primitive types are escaped accordingly. Python expressions are written as-is; -//! invalid expressions produce invalid configuration files. Config overrides that do -//! not map to a known option are treated as plain expressions. -//! -// TODO: This file is vendored identically in airflow-operator and superset-operator; -// it is a candidate for a shared crate (e.g. operator-rs) so the duplication can be -// removed. Until such a home exists it is kept identical in both operators and to the -// upstream source. - -use std::{ - io::{self, Write}, - num::ParseIntError, - str::{FromStr, ParseBoolError}, -}; - -use snafu::{ResultExt, Snafu}; - -/// Errors which can occur when using this module -// Variant names share the `Error` suffix; kept as-is from the vendored -// `product_config::flask_app_config_writer` source. -#[allow(clippy::enum_variant_names)] -#[derive(Debug, Snafu)] -pub enum FlaskAppConfigWriterError { - #[snafu(display("failed to convert '{value}' into a identifier"))] - ConvertIdentifierError { value: String }, - - #[snafu(display("failed to convert '{value}' into a boolean literal"))] - ConvertBoolLiteralError { - value: String, - source: ParseBoolError, - }, - - #[snafu(display("failed to convert '{value}' into an integer literal"))] - ConvertIntLiteralError { - value: String, - source: ParseIntError, - }, - - #[snafu(display("failed to convert '{value}' into an ASCII string literal"))] - ConvertStringLiteralError { value: String }, - - #[snafu(display("failed to convert '{value}' into a Python expression"))] - ConvertExpressionError { value: String }, - - #[snafu(display("Configuration cannot be written."))] - WriteConfigError { source: io::Error }, -} - -/// Mapping from configuration options to Python types. -pub trait FlaskAppConfigOptions { - fn python_type(&self) -> PythonType; -} - -/// All supported Python types -pub enum PythonType { - /// Python identifier - Identifier, - /// Boolean literal - BoolLiteral, - /// Integer literal - IntLiteral, - /// ASCII string literal - StringLiteral, - /// Python expression - Expression, -} - -impl PythonType { - /// Converts the given string to Python. - fn convert_to_python(&self, value: &str) -> Result { - let convert = match self { - PythonType::Identifier => PythonType::convert_to_python_identifier, - PythonType::BoolLiteral => PythonType::convert_to_python_bool_literal, - PythonType::IntLiteral => PythonType::convert_to_python_int_literal, - PythonType::StringLiteral => PythonType::convert_to_python_string_literal, - PythonType::Expression => PythonType::convert_to_python_expression, - }; - - convert(value) - } - - fn convert_to_python_identifier(value: &str) -> Result { - if value.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') - && value - .chars() - .next() - .filter(|c| !c.is_ascii_digit()) - .is_some() - { - Ok(value.to_string()) - } else { - ConvertIdentifierSnafu { value }.fail() - } - } - - fn convert_to_python_bool_literal(value: &str) -> Result { - value - .parse::() - .map(|b| if b { "True".into() } else { "False".into() }) - .context(ConvertBoolLiteralSnafu { value }) - } - - fn convert_to_python_int_literal(value: &str) -> Result { - value - .parse::() - .map(|i| i.to_string()) - .context(ConvertIntLiteralSnafu { value }) - } - - fn convert_to_python_string_literal(value: &str) -> Result { - if value.is_ascii() { - Ok(format!("\"{}\"", value.escape_default())) - } else { - ConvertStringLiteralSnafu { value }.fail() - } - } - - fn convert_to_python_expression(value: &str) -> Result { - if !value.trim().is_empty() { - Ok(value.to_string()) - } else { - ConvertExpressionSnafu { value }.fail() - } - } -} - -/// Writes a configuration file according to the given `FlaskAppConfigOptions` type. -pub fn write<'a, O, P, W>( - writer: &mut W, - properties: P, - imports: &[&str], -) -> Result<(), FlaskAppConfigWriterError> -where - O: FlaskAppConfigOptions + FromStr, - P: Iterator, - W: Write, -{ - for import in imports { - writeln!(writer, "{import}").context(WriteConfigSnafu)?; - } - - writeln!(writer).context(WriteConfigSnafu)?; - - for (name, value) in properties { - let variable = PythonType::Identifier.convert_to_python(name)?; - - // If an option cannot be mapped to a Python type then it is a config override and treated - // as Python expression. - let content = O::from_str(name) - .map(|option| option.python_type()) - .unwrap_or(PythonType::Expression) - .convert_to_python(value)?; - - writeln!(writer, "{variable} = {content}").context(WriteConfigSnafu)?; - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use std::{ - collections::BTreeMap, - str::{FromStr, from_utf8}, - }; - - use rstest::*; - - use super::{FlaskAppConfigOptions, FlaskAppConfigWriterError, PythonType, write}; - - #[rstest] - #[case::valid_identifiers_are_converted_to_python( - PythonType::Identifier, &[ - ("_", "_"), - ("a", "a"), - ("A", "A"), - ("__", "__"), - ("_a", "_a"), - ("_A", "_A"), - ("_0", "_0"), - ("SECRET_KEY", "SECRET_KEY"), - ] - )] - #[case::valid_booleans_are_converted_to_python( - PythonType::BoolLiteral, &[ - ("False", "false"), - ("True", "true"), - ] - )] - #[case::valid_integers_are_converted_to_python( - PythonType::IntLiteral, &[ - ("-9223372036854775808", "-9223372036854775808"), - ("0", "0"), - ("9223372036854775807", "9223372036854775807"), - ] - )] - #[case::valid_strings_are_converted_to_python( - PythonType::StringLiteral, &[ - (r#""""#, ""), - (r#"" ~""#, " ~"), - (r#""\t\r\n\'\"\\""#, "\t\r\n'\"\\"), - ] - )] - #[case::valid_expressions_are_converted_to_python( - PythonType::Expression, &[ - ("os.environ[\"HOME\"]", "os.environ[\"HOME\"]"), - ] - )] - fn valid_values_are_converted_to_python( - #[case] python_type: PythonType, - #[case] values: &[(&str, &str)], - ) -> Result<(), FlaskAppConfigWriterError> { - for (expected, input) in values { - assert_eq!(*expected, python_type.convert_to_python(input)?); - } - - Ok(()) - } - - #[rstest] - #[case::invalid_identifiers_are_not_converted_to_python( - PythonType::Identifier, &[ - "", "0", "-", "\n", "_-", "_\n", - ] - )] - #[case::invalid_booleans_are_not_converted_to_python( - PythonType::BoolLiteral, &[ - "", "False", "True", "0", "1", - ] - )] - #[case::invalid_integers_are_not_converted_to_python( - PythonType::IntLiteral, &[ - "", "a", "0x10", "inf", - ] - )] - #[case::invalid_strings_are_not_converted_to_python( - PythonType::StringLiteral, &[ - "ä", "❤" - ] - )] - #[case::invalid_expressions_are_not_converted_to_python( - PythonType::Expression, &[ - "" - ] - )] - fn invalid_values_are_converted_to_python( - #[case] python_type: PythonType, - #[case] values: &[&str], - ) { - for input in values { - assert!(python_type.convert_to_python(input).is_err()); - } - } - - #[test] - fn valid_options_are_written_into_a_configuration() -> Result<(), FlaskAppConfigWriterError> { - #[allow(clippy::enum_variant_names)] - enum Options { - BoolOption, - IntOption, - StringOption, - ExpressionOption, - _UnusedOption, - } - - impl FromStr for Options { - type Err = &'static str; - - fn from_str(s: &str) -> Result { - match s { - "BOOL_OPTION" => Ok(Options::BoolOption), - "INT_OPTION" => Ok(Options::IntOption), - "STRING_OPTION" => Ok(Options::StringOption), - "EXPRESSION_OPTION" => Ok(Options::ExpressionOption), - _ => Err("unknown option"), - } - } - } - - impl FlaskAppConfigOptions for Options { - fn python_type(&self) -> PythonType { - match self { - Options::BoolOption => PythonType::BoolLiteral, - Options::IntOption => PythonType::IntLiteral, - Options::StringOption => PythonType::StringLiteral, - Options::ExpressionOption => PythonType::Expression, - Options::_UnusedOption => PythonType::Expression, - } - } - } - - let config: BTreeMap<_, _> = [ - ("BOOL_OPTION", "true"), - ("INT_OPTION", "0"), - ("STRING_OPTION", ""), - ("EXPRESSION_OPTION", "{ \"key\": \"value\" }"), - ("OVERRIDDEN_OPTION", "None"), - ] - .map(|(k, v)| (k.to_string(), v.to_string())) - .into(); - - let imports = ["import module", "from module import member"]; - - let mut config_file = Vec::new(); - write::(&mut config_file, config.iter(), &imports)?; - - assert_eq!( - r#"import module -from module import member - -BOOL_OPTION = True -EXPRESSION_OPTION = { "key": "value" } -INT_OPTION = 0 -OVERRIDDEN_OPTION = None -STRING_OPTION = "" -"#, - from_utf8(&config_file).unwrap() - ); - - Ok(()) - } -} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c28543e4..e7b228d6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -39,13 +39,15 @@ use stackable_operator::{ shared::time::Duration, status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, - v2::config_overrides::KeyValueConfigOverrides, + v2::{ + config_overrides::KeyValueConfigOverrides, + flask_config_writer::{FlaskAppConfigOptions, PythonType}, + }, versioned::versioned, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; use crate::{ - config::writer::{FlaskAppConfigOptions, PythonType}, crd::{ affinity::{get_affinity, get_executor_affinity}, authentication::{