From 3391f5eff10076d07a7409119fdb0b617aa7c688 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Feb 2026 21:35:43 +0000 Subject: [PATCH] Implement JIT query rendering to avoid rendering unused templates Previously, get_queries() rendered ALL query templates upfront (exists, create, update, delete, exports, etc.) regardless of which operation was being run. This caused failures when a template referenced variables not yet available in the context - e.g., a delete query referencing an export variable (databricks_credentials_id) that only gets populated after the exports query runs. Changed to JIT (just-in-time) rendering: - get_queries() now only loads and parses templates without rendering - ParsedQuery stores the raw template instead of a pre-rendered string - Each command (build, teardown, test) renders templates on demand via render_query() only when the query is actually needed - Added render_query() to CommandRunner for convenient JIT rendering This matches how the Python version works in practice - Jinja2 is more forgiving with missing variables, but the proper fix is to only render what you need, when you need it. https://claude.ai/code/session_01ShAyjRLWBYC3tPsusCxggv --- src/commands/base.rs | 11 ++++ src/commands/build.rs | 122 +++++++++++++++++++++++---------------- src/commands/teardown.rs | 30 +++++----- src/commands/test.rs | 30 +++++++--- src/core/templating.rs | 74 +++++++++++++----------- 5 files changed, 158 insertions(+), 109 deletions(-) diff --git a/src/commands/base.rs b/src/commands/base.rs index d4fdd23..03a7056 100644 --- a/src/commands/base.rs +++ b/src/commands/base.rs @@ -144,6 +144,17 @@ impl CommandRunner { templating::render_inline_template(&self.engine, resource_name, sql, full_context) } + /// Render a single query template JIT with the current context. + pub fn render_query( + &self, + resource_name: &str, + anchor: &str, + template: &str, + full_context: &HashMap, + ) -> String { + templating::render_query(&self.engine, resource_name, anchor, template, full_context) + } + /// Check if a resource exists using the exists query. #[allow(clippy::too_many_arguments)] pub fn check_if_resource_exists( diff --git a/src/commands/build.rs b/src/commands/build.rs index 00030d9..427c3c0 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -140,7 +140,7 @@ fn run_build( continue; } - // Get resource queries + // Get resource queries (templates only, not yet rendered) let (resource_queries, inline_query) = if let Some(sql_val) = resource .sql .as_ref() @@ -164,20 +164,36 @@ fn run_build( if res_type == "resource" || res_type == "multi" { if let Some(cou) = resource_queries.get("createorupdate") { has_createorupdate = true; - create_query = Some(cou.rendered.clone()); + let rendered = runner.render_query( + &resource.name, + "createorupdate", + &cou.template, + &full_context, + ); + create_query = Some(rendered.clone()); create_retries = cou.options.retries; create_retry_delay = cou.options.retry_delay; - update_query = Some(cou.rendered.clone()); + update_query = Some(rendered); update_retries = cou.options.retries; update_retry_delay = cou.options.retry_delay; } else { if let Some(cq) = resource_queries.get("create") { - create_query = Some(cq.rendered.clone()); + create_query = Some(runner.render_query( + &resource.name, + "create", + &cq.template, + &full_context, + )); create_retries = cq.options.retries; create_retry_delay = cq.options.retry_delay; } if let Some(uq) = resource_queries.get("update") { - update_query = Some(uq.rendered.clone()); + update_query = Some(runner.render_query( + &resource.name, + "update", + &uq.template, + &full_context, + )); update_retries = uq.options.retries; update_retry_delay = uq.options.retry_delay; } @@ -190,11 +206,20 @@ fn run_build( } } - // Test queries - let exists_query = resource_queries.get("exists"); - let statecheck_query = resource_queries.get("statecheck"); - let mut exports_query_str: Option = - resource_queries.get("exports").map(|q| q.rendered.clone()); + // Test queries - render only the ones we need + let exists_query = resource_queries.get("exists").map(|q| { + let rendered = + runner.render_query(&resource.name, "exists", &q.template, &full_context); + (rendered, q.options.clone()) + }); + let statecheck_query = resource_queries.get("statecheck").map(|q| { + let rendered = + runner.render_query(&resource.name, "statecheck", &q.template, &full_context); + (rendered, q.options.clone()) + }); + let mut exports_query_str: Option = resource_queries + .get("exports") + .map(|q| runner.render_query(&resource.name, "exports", &q.template, &full_context)); let exports_opts = resource_queries.get("exports"); let exports_retries = exports_opts.map_or(1, |q| q.options.retries); let exports_retry_delay = exports_opts.map_or(0, |q| q.options.retry_delay); @@ -222,24 +247,26 @@ fn run_build( // Skip all existence and state checks for createorupdate } else if statecheck_query.is_some() { // Flow 1: Traditional flow when statecheck exists - if let Some(eq) = exists_query { + if let Some(ref eq) = exists_query { + let eq_opts = resource_queries.get("exists").unwrap(); resource_exists = runner.check_if_resource_exists( resource, - &eq.rendered, - eq.options.retries, - eq.options.retry_delay, + &eq.0, + eq_opts.options.retries, + eq_opts.options.retry_delay, dry_run, show_queries, false, ); } else { // Use statecheck as exists check - let sq = statecheck_query.unwrap(); + let sq = statecheck_query.as_ref().unwrap(); + let sq_opts = resource_queries.get("statecheck").unwrap(); is_correct_state = runner.check_if_resource_is_correct_state( resource, - &sq.rendered, - sq.options.retries, - sq.options.retry_delay, + &sq.0, + sq_opts.options.retries, + sq_opts.options.retry_delay, dry_run, show_queries, ); @@ -255,18 +282,19 @@ fn run_build( ); is_correct_state = true; } else { - let sq = statecheck_query.unwrap(); + let sq = statecheck_query.as_ref().unwrap(); + let sq_opts = resource_queries.get("statecheck").unwrap(); is_correct_state = runner.check_if_resource_is_correct_state( resource, - &sq.rendered, - sq.options.retries, - sq.options.retry_delay, + &sq.0, + sq_opts.options.retries, + sq_opts.options.retry_delay, dry_run, show_queries, ); } } - } else if let Some(eq_str) = exports_query_str.as_ref() { + } else if let Some(ref eq_str) = exports_query_str { // Flow 2: Optimized flow using exports as proxy info!( "trying exports query first (fast-fail) for optimal validation for [{}]", @@ -296,12 +324,13 @@ fn run_build( ); exports_result_from_proxy = None; - if let Some(eq) = exists_query { + if let Some(ref eq) = exists_query { + let eq_opts = resource_queries.get("exists").unwrap(); resource_exists = runner.check_if_resource_exists( resource, - &eq.rendered, - eq.options.retries, - eq.options.retry_delay, + &eq.0, + eq_opts.options.retries, + eq_opts.options.retry_delay, dry_run, show_queries, false, @@ -310,13 +339,14 @@ fn run_build( resource_exists = false; } } - } else if let Some(eq) = exists_query { + } else if let Some(ref eq) = exists_query { // Flow 3: Basic flow with only exists query + let eq_opts = resource_queries.get("exists").unwrap(); resource_exists = runner.check_if_resource_exists( resource, - &eq.rendered, - eq.options.retries, - eq.options.retry_delay, + &eq.0, + eq_opts.options.retries, + eq_opts.options.retry_delay, dry_run, show_queries, false, @@ -356,12 +386,13 @@ fn run_build( // Post-deploy state check if is_created_or_updated { - if let Some(sq) = statecheck_query { + if let Some(ref sq) = statecheck_query { + let sq_opts = resource_queries.get("statecheck").unwrap(); is_correct_state = runner.check_if_resource_is_correct_state( resource, - &sq.rendered, - sq.options.retries, - sq.options.retry_delay, + &sq.0, + sq_opts.options.retries, + sq_opts.options.retry_delay, dry_run, show_queries, ); @@ -370,17 +401,8 @@ fn run_build( "using exports query as post-deploy statecheck for [{}]", resource.name ); - let post_retries = if statecheck_query.is_some_and(|sq| sq.options.retries > 1) - { - statecheck_query.unwrap().options.retries - } else { - exports_retries - }; - let post_delay = if statecheck_query.is_some_and(|sq| sq.options.retries > 1) { - statecheck_query.unwrap().options.retry_delay - } else { - exports_retry_delay - }; + let post_retries = exports_retries; + let post_delay = exports_retry_delay; let (state, proxy) = runner.check_state_using_exports_proxy( resource, @@ -412,11 +434,9 @@ fn run_build( { (iq.clone(), 1u32, 0u32) } else if let Some(cq) = resource_queries.get("command") { - ( - cq.rendered.clone(), - cq.options.retries, - cq.options.retry_delay, - ) + let rendered = + runner.render_query(&resource.name, "command", &cq.template, &full_context); + (rendered, cq.options.retries, cq.options.retry_delay) } else { catch_error_and_exit( "'sql' should be defined in the resource or the 'command' anchor needs to be supplied in the corresponding iql file for command type resources.", diff --git a/src/commands/teardown.rs b/src/commands/teardown.rs index 75f28b8..b756b34 100644 --- a/src/commands/teardown.rs +++ b/src/commands/teardown.rs @@ -108,11 +108,9 @@ fn collect_exports(runner: &mut CommandRunner, show_queries: bool, dry_run: bool } else { let queries = runner.get_queries(resource, &full_context); if let Some(eq) = queries.get("exports") { - ( - Some(eq.rendered.clone()), - eq.options.retries, - eq.options.retry_delay, - ) + let rendered = + runner.render_query(&resource.name, "exports", &eq.template, &full_context); + (Some(rendered), eq.options.retries, eq.options.retry_delay) } else { (None, 1u32, 0u32) } @@ -195,10 +193,10 @@ fn run_teardown(runner: &mut CommandRunner, dry_run: bool, show_queries: bool, _ } } - // Get resource queries + // Get resource queries (templates only) let resource_queries = runner.get_queries(resource, &full_context); - // Get exists query (fallback to statecheck) + // Get exists query (fallback to statecheck) - render JIT let ( exists_query_str, exists_retries, @@ -206,8 +204,10 @@ fn run_teardown(runner: &mut CommandRunner, dry_run: bool, show_queries: bool, _ postdelete_retries, postdelete_retry_delay, ) = if let Some(eq) = resource_queries.get("exists") { + let rendered = + runner.render_query(&resource.name, "exists", &eq.template, &full_context); ( - eq.rendered.clone(), + rendered, eq.options.retries, eq.options.retry_delay, eq.options.postdelete_retries, @@ -218,8 +218,10 @@ fn run_teardown(runner: &mut CommandRunner, dry_run: bool, show_queries: bool, _ "exists query not defined for [{}], trying statecheck query as exists query.", resource.name ); + let rendered = + runner.render_query(&resource.name, "statecheck", &sq.template, &full_context); ( - sq.rendered.clone(), + rendered, sq.options.retries, sq.options.retry_delay, sq.options.postdelete_retries, @@ -233,14 +235,12 @@ fn run_teardown(runner: &mut CommandRunner, dry_run: bool, show_queries: bool, _ continue; }; - // Get delete query + // Get delete query - render JIT let (delete_query, delete_retries, delete_retry_delay) = if let Some(dq) = resource_queries.get("delete") { - ( - dq.rendered.clone(), - dq.options.retries, - dq.options.retry_delay, - ) + let rendered = + runner.render_query(&resource.name, "delete", &dq.template, &full_context); + (rendered, dq.options.retries, dq.options.retry_delay) } else { info!( "delete query not defined for [{}], skipping...", diff --git a/src/commands/test.rs b/src/commands/test.rs index 535e5f3..8b6024c 100644 --- a/src/commands/test.rs +++ b/src/commands/test.rs @@ -131,7 +131,7 @@ fn run_test( let full_context = runner.get_full_context(resource); - // Get test queries + // Get test queries (templates only, not yet rendered) let (test_queries, inline_query) = if let Some(sql_val) = resource.sql.as_ref().filter(|_| res_type == "query") { let iq = runner.render_inline_template(&resource.name, sql_val, &full_context); @@ -140,11 +140,23 @@ fn run_test( (runner.get_queries(resource, &full_context), None) }; - let statecheck_query = test_queries.get("statecheck"); - let statecheck_retries = statecheck_query.map_or(1, |q| q.options.retries); - let statecheck_retry_delay = statecheck_query.map_or(0, |q| q.options.retry_delay); + // Render statecheck JIT if present + let statecheck_rendered = test_queries.get("statecheck").map(|q| { + let rendered = + runner.render_query(&resource.name, "statecheck", &q.template, &full_context); + (rendered, q.options.clone()) + }); + let statecheck_retries = test_queries + .get("statecheck") + .map_or(1, |q| q.options.retries); + let statecheck_retry_delay = test_queries + .get("statecheck") + .map_or(0, |q| q.options.retry_delay); - let mut exports_query_str = test_queries.get("exports").map(|q| q.rendered.clone()); + // Render exports JIT if present + let mut exports_query_str = test_queries + .get("exports") + .map(|q| runner.render_query(&resource.name, "exports", &q.template, &full_context)); let exports_opts = test_queries.get("exports"); let exports_retries = exports_opts.map_or(1, |q| q.options.retries); let exports_retry_delay = exports_opts.map_or(0, |q| q.options.retry_delay); @@ -168,12 +180,12 @@ fn run_test( if resource.skip_validation.unwrap_or(false) { info!("Skipping statecheck for {}", resource.name); is_correct_state = true; - } else if let Some(sq) = statecheck_query { + } else if let Some(ref sq) = statecheck_rendered { is_correct_state = runner.check_if_resource_is_correct_state( resource, - &sq.rendered, - sq.options.retries, - sq.options.retry_delay, + &sq.0, + sq.1.retries, + sq.1.retry_delay, dry_run, show_queries, ); diff --git a/src/core/templating.rs b/src/core/templating.rs index 917f069..7466435 100644 --- a/src/core/templating.rs +++ b/src/core/templating.rs @@ -4,6 +4,11 @@ //! //! Handles loading, parsing, and rendering SQL query templates from .iql files. //! Matches the Python `lib/templating.py` implementation. +//! +//! Queries are loaded and parsed eagerly, but rendered lazily (JIT) when +//! actually needed. This avoids errors from templates that reference variables +//! not yet available in the context (e.g., delete queries referencing exports +//! that haven't been computed yet during a build operation). use std::collections::HashMap; use std::fs; @@ -17,11 +22,11 @@ use crate::core::config::prepare_query_context; use crate::resource::manifest::Resource; use crate::template::engine::TemplateEngine; -/// Parsed query with its rendered form and options. +/// Parsed query with its raw template and options. +/// Rendering is deferred until the query is actually needed. #[derive(Debug, Clone)] pub struct ParsedQuery { pub template: String, - pub rendered: String, pub options: QueryOptions, } @@ -122,7 +127,7 @@ fn load_sql_queries( /// becomes: /// `{{ __inline_dict_0 | generate_patch_document }}` /// with `__inline_dict_0` set to the constructed JSON object in context. -fn preprocess_inline_dicts(template: &str, context: &mut HashMap) -> String { +pub fn preprocess_inline_dicts(template: &str, context: &mut HashMap) -> String { // Match {{ { ... } | filter_name }} // This regex captures the dict body and the filter expression let re = Regex::new(r"\{\{\s*\{([^}]*(?:\{[^}]*\}[^}]*)*)\}\s*\|\s*(\w+)\s*\}\}").unwrap(); @@ -227,49 +232,53 @@ fn split_dict_entries(s: &str) -> Vec { entries } -/// Render query templates with the full context. -/// Matches Python's `render_queries`. -fn render_queries( +/// Render a single query template with the given context. +/// This is the JIT rendering function called when a query is actually needed. +pub fn render_query( engine: &TemplateEngine, res_name: &str, - queries: &HashMap, + anchor: &str, + template: &str, context: &HashMap, -) -> HashMap { - let mut rendered_queries = HashMap::new(); - - // Prepare context: re-serialize JSON values +) -> String { let temp_context = prepare_query_context(context); - for (key, query) in queries { - debug!("[{}] [{}] query template:\n\n{}\n", res_name, key, query); + debug!( + "[{}] [{}] query template:\n\n{}\n", + res_name, anchor, template + ); - // Pre-process inline dict expressions and render with filters - let mut ctx = temp_context.clone(); - let processed_query = preprocess_inline_dicts(query, &mut ctx); + let mut ctx = temp_context; + let processed_query = preprocess_inline_dicts(template, &mut ctx); - let template_name = format!("{}__{}", res_name, key); - match engine.render_with_filters(&template_name, &processed_query, &ctx) { - Ok(rendered) => { - debug!("[{}] [{}] rendered query:\n\n{}\n", res_name, key, rendered); - rendered_queries.insert(key.clone(), rendered); - } - Err(e) => { - error!("Error rendering query for [{}] [{}]: {}", res_name, key, e); - process::exit(1); - } + let template_name = format!("{}__{}", res_name, anchor); + match engine.render_with_filters(&template_name, &processed_query, &ctx) { + Ok(rendered) => { + debug!( + "[{}] [{}] rendered query:\n\n{}\n", + res_name, anchor, rendered + ); + rendered + } + Err(e) => { + error!( + "Error rendering query for [{}] [{}]: {}", + res_name, anchor, e + ); + process::exit(1); } } - - rendered_queries } -/// Get queries for a resource: load from file, parse anchors, render with context. +/// Get queries for a resource: load from file, parse anchors. +/// Templates are NOT rendered here — rendering is deferred to when +/// each query is actually needed (JIT rendering). /// Matches Python's `get_queries`. pub fn get_queries( - engine: &TemplateEngine, + _engine: &TemplateEngine, stack_dir: &str, resource: &Resource, - full_context: &HashMap, + _full_context: &HashMap, ) -> HashMap { let mut result = HashMap::new(); @@ -287,7 +296,6 @@ pub fn get_queries( } let (query_templates, query_options) = load_sql_queries(&template_path); - let rendered_queries = render_queries(engine, &resource.name, &query_templates, full_context); for (anchor, template) in &query_templates { // Fix backward compatibility for preflight and postdeploy @@ -298,13 +306,11 @@ pub fn get_queries( }; let opts = query_options.get(anchor).cloned().unwrap_or_default(); - let rendered = rendered_queries.get(anchor).cloned().unwrap_or_default(); result.insert( normalized_anchor.clone(), ParsedQuery { template: template.clone(), - rendered, options: QueryOptions { retries: *opts.get("retries").unwrap_or(&1), retry_delay: *opts.get("retry_delay").unwrap_or(&0),