From bba999853e04a2cadf1392dad2dbeed82060fbff Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 23 Feb 2026 04:30:07 +0000 Subject: [PATCH] Add file() directive documentation and fix cargo fmt - Add comprehensive documentation page for the file() directive with examples covering JSON/YAML includes, list items, whole value replacement, globals, subdirectories, and nested directives - Add file() directive to the sidebar navigation - Update resource.prop.value docs with a tip about file() usage - Fix cargo fmt formatting issues in manifest.rs https://claude.ai/code/session_01FikwxWpZdvhtNbECVWJiqK --- src/resource/manifest.rs | 50 ++-- website/docs/file-directive.md | 268 ++++++++++++++++++ .../manifest_fields/resources/props/value.mdx | 23 +- website/sidebars.js | 9 +- 4 files changed, 325 insertions(+), 25 deletions(-) create mode 100644 website/docs/file-directive.md diff --git a/src/resource/manifest.rs b/src/resource/manifest.rs index 8f5b21b..a9b821d 100644 --- a/src/resource/manifest.rs +++ b/src/resource/manifest.rs @@ -239,7 +239,10 @@ fn resolve_file_directives(value: &mut serde_yaml::Value, base_dir: &Path) -> Ma fn load_file_contents(file_path: &str, base_dir: &Path) -> ManifestResult { let full_path = base_dir.join(file_path); - debug!("Resolving file() directive: {} -> {:?}", file_path, full_path); + debug!( + "Resolving file() directive: {} -> {:?}", + file_path, full_path + ); let content = fs::read_to_string(&full_path).map_err(|e| { ManifestError::FileIncludeError(format!( @@ -248,20 +251,16 @@ fn load_file_contents(file_path: &str, base_dir: &Path) -> ManifestResult { - let json_val: serde_json::Value = - serde_json::from_str(&content).map_err(|e| { - ManifestError::FileIncludeError(format!( - "failed to parse JSON file '{}': {}", - file_path, e - )) - })?; + let json_val: serde_json::Value = serde_json::from_str(&content).map_err(|e| { + ManifestError::FileIncludeError(format!( + "failed to parse JSON file '{}': {}", + file_path, e + )) + })?; // Convert JSON -> YAML value serde_yaml::to_value(&json_val).map_err(|e| { ManifestError::FileIncludeError(format!( @@ -469,8 +468,14 @@ mod tests { #[test] fn test_parse_file_directive() { - assert_eq!(parse_file_directive("file(foo/bar.json)"), Some("foo/bar.json")); - assert_eq!(parse_file_directive(" file( foo/bar.json ) "), Some("foo/bar.json")); + assert_eq!( + parse_file_directive("file(foo/bar.json)"), + Some("foo/bar.json") + ); + assert_eq!( + parse_file_directive(" file( foo/bar.json ) "), + Some("foo/bar.json") + ); assert_eq!(parse_file_directive("file()"), None); assert_eq!(parse_file_directive("not a directive"), None); assert_eq!(parse_file_directive("file("), None); @@ -564,11 +569,7 @@ mod tests { fn test_resolve_file_directive_nested_in_mapping() { let dir = setup_test_dir(); let resources_dir = dir.path().join("resources"); - fs::write( - resources_dir.join("stmts.json"), - r#"[{"Effect": "Allow"}]"#, - ) - .unwrap(); + fs::write(resources_dir.join("stmts.json"), r#"[{"Effect": "Allow"}]"#).unwrap(); let mut map = serde_yaml::Mapping::new(); map.insert( @@ -628,7 +629,10 @@ mod tests { let mut value = serde_yaml::Value::String("just a normal string".to_string()); resolve_file_directives(&mut value, &resources_dir).unwrap(); - assert_eq!(value, serde_yaml::Value::String("just a normal string".into())); + assert_eq!( + value, + serde_yaml::Value::String("just a normal string".into()) + ); } #[test] @@ -679,7 +683,11 @@ resources: let manifest = Manifest::load_from_stack_dir(dir.path()).unwrap(); let resource = manifest.find_resource("test_resource").unwrap(); - let policies_prop = resource.props.iter().find(|p| p.name == "policies").unwrap(); + let policies_prop = resource + .props + .iter() + .find(|p| p.name == "policies") + .unwrap(); let value = policies_prop.value.as_ref().unwrap(); // The value should be a sequence with one policy diff --git a/website/docs/file-directive.md b/website/docs/file-directive.md new file mode 100644 index 0000000..fb2ac60 --- /dev/null +++ b/website/docs/file-directive.md @@ -0,0 +1,268 @@ +--- +id: file-directive +title: file() Directive +hide_title: false +hide_table_of_contents: false +description: Use the file() directive to modularize manifest values by including external JSON or YAML files +tags: [] +draft: false +unlisted: false +--- + +import File from '/src/components/File'; + +The `file()` directive allows you to include the contents of external JSON or YAML files directly into your `stackql_manifest.yml` property values. This is particularly useful for modularizing large or reusable configuration blocks like IAM policy statements, role definitions, or any complex nested structures. + +## Syntax + +```yaml +file(relative/path/to/file.json) +``` + +The argument is a file path **relative to the `resources/` directory** of your stack (the same base path used for `.iql` resource query files). + +## Supported file formats + +| Extension | Format | +|---|---| +| `.json` | Parsed as JSON | +| `.yml`, `.yaml` | Parsed as YAML | +| Other/none | Tries JSON first, falls back to YAML | + +## How it works + +When the manifest is loaded, the `file()` directive is resolved **before** any template variable substitution occurs. The referenced file is read and parsed, and the resulting value (object, array, or scalar) replaces the `file()` string in the manifest value tree. + +:::note + +Because `file()` directives are resolved at manifest load time (before template rendering), the included files should contain static data only. Template variables like `{{ stack_name }}` in property values surrounding the `file()` directive will still be rendered as normal. + +::: + +## Usage + +### Including individual items in a list + +You can use `file()` as individual items within a YAML sequence. Each directive is replaced by the parsed contents of the referenced file. + + + +```yaml +resources: + - name: aws/iam/cross_account_role + file: aws/iam/iam_role.iql + props: + - name: policies + value: + - PolicyDocument: + Statement: + - file(aws/iam/policies/ec2_permissions.json) + - file(aws/iam/policies/iam_service_linked_role.json) + Version: '2012-10-17' + PolicyName: "{{ stack_name }}-{{ stack_env }}-policy" +``` + + + +Where `resources/aws/iam/policies/ec2_permissions.json` contains a single policy statement object: + + + +```json +{ + "Sid": "Stmt1403287045000", + "Effect": "Allow", + "Action": [ + "ec2:AllocateAddress", + "ec2:AssociateDhcpOptions", + "ec2:AssociateIamInstanceProfile", + "ec2:AssociateRouteTable", + "ec2:AttachInternetGateway", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVpc", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteVpc", + "ec2:DescribeInstances", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVpcs", + "ec2:RunInstances", + "ec2:TerminateInstances" + ], + "Resource": ["*"] +} +``` + + + +And `resources/aws/iam/policies/iam_service_linked_role.json`: + + + +```json +{ + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole", + "iam:PutRolePolicy" + ], + "Resource": [ + "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot" + ], + "Condition": { + "StringLike": { + "iam:AWSServiceName": "spot.amazonaws.com" + } + } +} +``` + + + +### Replacing an entire value with a file + +You can also use `file()` to replace an entire value, such as a complete list of statements: + + + +```yaml +resources: + - name: aws/iam/cross_account_role + file: aws/iam/iam_role.iql + props: + - name: policies + value: + - PolicyDocument: + Statement: file(aws/iam/policies/cross_account_statements.json) + Version: '2012-10-17' + PolicyName: "{{ stack_name }}-{{ stack_env }}-policy" +``` + + + +Where `resources/aws/iam/policies/cross_account_statements.json` is a JSON **array**: + + + +```json +[ + { + "Sid": "Stmt1403287045000", + "Effect": "Allow", + "Action": ["ec2:AllocateAddress", "ec2:CreateVpc", "ec2:DeleteVpc"], + "Resource": ["*"] + }, + { + "Effect": "Allow", + "Action": ["iam:CreateServiceLinkedRole", "iam:PutRolePolicy"], + "Resource": ["arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"], + "Condition": { + "StringLike": { + "iam:AWSServiceName": "spot.amazonaws.com" + } + } + } +] +``` + + + +### Using YAML files + +YAML files can be used instead of JSON, which can be more readable for some configurations: + + + +```yaml +resources: + - name: aws/iam/metastore_access_role + file: aws/iam/iam_role.iql + props: + - name: policies + value: + - PolicyName: MetastoreS3Policy + PolicyDocument: + Version: '2012-10-17' + Statement: file(aws/iam/policies/metastore_statements.yaml) +``` + + + + + +```yaml +- Effect: Allow + Action: + - "s3:GetObject" + - "s3:PutObject" + - "s3:DeleteObject" + - "s3:ListBucket" + - "s3:GetBucketLocation" + Resource: + - "arn:aws:s3:::my-metastore-bucket/*" + - "arn:aws:s3:::my-metastore-bucket" +- Effect: Allow + Action: + - "sts:AssumeRole" + Resource: + - "arn:aws:iam::123456789012:role/my-metastore-role" +``` + + + +### Using in globals + +The `file()` directive also works in global variable values: + + + +```yaml +globals: + - name: default_tags + value: file(common/default_tags.json) +``` + + + +### Subdirectory organization + +File paths can include subdirectories, making it easy to organize included files alongside your resource query files: + +``` +my-stack/ + stackql_manifest.yml + resources/ + aws/ + iam/ + iam_role.iql + policies/ + ec2_permissions.json + iam_service_linked_role.json + metastore_statements.yaml + s3/ + s3_bucket.iql +``` + +### Nested file() directives + +Included files can themselves contain `file()` directives, which will be resolved recursively. This allows you to compose configurations from multiple reusable fragments: + + + +```yaml +Version: '2012-10-17' +Statement: + - file(aws/iam/policies/base_permissions.json) + - file(aws/iam/policies/extra_permissions.json) +``` + + + +## Error handling + +If a `file()` directive references a file that does not exist or contains invalid JSON/YAML, the manifest will fail to load with a descriptive error message indicating the problematic file path and the nature of the error. diff --git a/website/docs/manifest_fields/resources/props/value.mdx b/website/docs/manifest_fields/resources/props/value.mdx index 3d626f3..d4af8a5 100644 --- a/website/docs/manifest_fields/resources/props/value.mdx +++ b/website/docs/manifest_fields/resources/props/value.mdx @@ -13,8 +13,27 @@ The value for the property - name: public_address props: - name: address_name - value: "{{ stack_name }}-{{ stack_env }}-{{ region }}-ip-addr" + value: "{{ stack_name }}-{{ stack_env }}-{{ region }}-ip-addr" ... ``` - \ No newline at end of file + + +:::tip + +You can use the [`file()` directive](../../../file-directive) to include values from external JSON or YAML files. This is useful for modularizing large or reusable configuration blocks like IAM policy statements: + +```yaml +- name: policies + value: + - PolicyDocument: + Statement: + - file(aws/iam/policies/ec2_permissions.json) + - file(aws/iam/policies/service_linked_role.json) + Version: '2012-10-17' + PolicyName: "{{ stack_name }}-{{ stack_env }}-policy" +``` + +File paths are resolved relative to the `resources/` directory. See the [`file()` directive documentation](../../../file-directive) for more details. + +::: \ No newline at end of file diff --git a/website/sidebars.js b/website/sidebars.js index 380a793..c744887 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -37,9 +37,14 @@ const sidebars = { }, { type: 'doc', - id: 'manifest-file', + id: 'manifest-file', label: 'stackql_manifest.yml', - }, + }, + { + type: 'doc', + id: 'file-directive', + label: 'file() Directive', + }, { type: 'doc', id: 'resource-query-files',