From 86806632a3c80be07ffad1b88f089dd57a78711b Mon Sep 17 00:00:00 2001 From: blaginin Date: Tue, 10 Feb 2026 17:16:29 +0000 Subject: [PATCH 01/16] Runs-on for more actions --- .github/workflows/rust.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2fce6aeb5e5d3..9211a5cdeb1ce 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -45,7 +45,7 @@ jobs: # Check crate compiles and base cargo check passes linux-build-lib: name: linux build test - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -167,7 +167,7 @@ jobs: linux-cargo-check-datafusion: name: cargo check datafusion features needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -347,7 +347,7 @@ jobs: linux-test-example: name: cargo examples (amd64) needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -438,7 +438,7 @@ jobs: verify-benchmark-results: name: verify benchmark results (amd64) needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: From b3df1a2039491057347de99389d850ea533ca8db Mon Sep 17 00:00:00 2001 From: blaginin Date: Tue, 10 Feb 2026 17:21:54 +0000 Subject: [PATCH 02/16] add action as well --- .github/workflows/rust.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9211a5cdeb1ce..7ea7516effce0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -171,6 +171,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder @@ -351,6 +352,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -442,6 +444,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true From 8574af71218abb8f3a279c4ef91731f9f139bf3e Mon Sep 17 00:00:00 2001 From: blaginin Date: Tue, 10 Feb 2026 18:16:42 +0000 Subject: [PATCH 03/16] m8a? --- .github/workflows/rust.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 7ea7516effce0..879c3cf136ab7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -45,7 +45,7 @@ jobs: # Check crate compiles and base cargo check passes linux-build-lib: name: linux build test - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -167,7 +167,7 @@ jobs: linux-cargo-check-datafusion: name: cargo check datafusion features needs: linux-build-lib - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -268,7 +268,7 @@ jobs: linux-test: name: cargo test (amd64) needs: linux-build-lib - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust volumes: @@ -348,7 +348,7 @@ jobs: linux-test-example: name: cargo examples (amd64) needs: linux-build-lib - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -440,7 +440,7 @@ jobs: verify-benchmark-results: name: verify benchmark results (amd64) needs: linux-build-lib - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: From dc9034afa778654affa0995e9f6237193874bd33 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:10:51 +0000 Subject: [PATCH 04/16] Switch more actions --- .github/workflows/rust.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 87136cfc9b71b..8f58ec5509ebf 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -99,7 +99,7 @@ jobs: linux-datafusion-substrait-features: name: cargo check datafusion-substrait features needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -136,7 +136,7 @@ jobs: linux-datafusion-proto-features: name: cargo check datafusion-proto features needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -319,7 +319,7 @@ jobs: linux-test-datafusion-cli: name: cargo test datafusion-cli (amd64) needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -379,7 +379,7 @@ jobs: linux-test-doc: name: cargo test doc (amd64) needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -400,7 +400,7 @@ jobs: linux-rustdoc: name: cargo doc needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -474,7 +474,7 @@ jobs: sqllogictest-postgres: name: "Run sqllogictest with Postgres runner" needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust services: @@ -512,7 +512,7 @@ jobs: sqllogictest-substrait: name: "Run sqllogictest in Substrait round-trip mode" needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -642,7 +642,7 @@ jobs: clippy: name: clippy needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -688,7 +688,7 @@ jobs: config-docs-check: name: check configs.md and ***_functions.md is up-to-date needs: linux-build-lib - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: From a9314b94046fdbcb6158ed6a3c49c7a29c4ee2d9 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:29:32 +0000 Subject: [PATCH 05/16] runs-on action --- .github/workflows/rust.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8f58ec5509ebf..10ec29562bc4f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +# For some actions, we use Runs-On to run them on ASF infrastructure: https://datafusion.apache.org/contributor-guide/#ci-runners + name: Rust concurrency: @@ -140,6 +142,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder @@ -321,6 +324,7 @@ jobs: needs: linux-build-lib runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -383,6 +387,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -404,6 +409,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Rust toolchain uses: ./.github/actions/setup-builder @@ -492,6 +498,7 @@ jobs: --health-timeout 5s --health-retries 5 steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -516,6 +523,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -646,6 +654,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true @@ -692,6 +701,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true From 5f59f33cd3709aeb8bd9f4ef66571db8e6cbab1a Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:29:36 +0000 Subject: [PATCH 06/16] better docs --- docs/source/contributor-guide/index.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md index 900df2f88174f..17c2b2d7de204 100644 --- a/docs/source/contributor-guide/index.md +++ b/docs/source/contributor-guide/index.md @@ -204,4 +204,16 @@ It's recommended to write a high-quality issue with a clear problem statement an We use [Runs-On](https://runs-on.com/) for some actions in the main repository, which run in the ASF AWS account to speed up CI time. In forks, these actions run on the default GitHub runners since forks do not have access to ASF infrastructure. -We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. +To configure them, we use the following format: + +`runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}` + +This is a conditional expression that uses Runs-On custom runners for the main repository but falls back to the standard GitHub runners for forks. Runs-On configuration follows the [Runs-On pattern](https://runs-on.com/configuration/job-labels/). + +For those actions we also use the [Runs-On action](https://runs-on.com/caching/magic-cache/#how-to-use), which adds support for external caching and reports job metrics: + +`- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e` + +For the standard GitHub runners, this action will do nothing. + +We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. \ No newline at end of file From 73c1f5217153602c241575c5a832261c00a8c847 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:35:26 +0000 Subject: [PATCH 07/16] do not assert exact region --- datafusion-cli/src/object_storage.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion-cli/src/object_storage.rs b/datafusion-cli/src/object_storage.rs index 3cee78a5b17cc..1343d142b70ba 100644 --- a/datafusion-cli/src/object_storage.rs +++ b/datafusion-cli/src/object_storage.rs @@ -749,7 +749,6 @@ mod tests { eprintln!("{e}"); return Ok(()); } - let expected_region = "eu-central-1"; let location = "s3://test-bucket/path/file.parquet"; // Set it to a non-existent file to avoid reading the default configuration file unsafe { @@ -766,9 +765,10 @@ mod tests { get_s3_object_store_builder(table_url.as_ref(), &aws_options, false).await?; // Verify that the region was auto-detected in test environment - assert_eq!( - builder.get_config_value(&AmazonS3ConfigKey::Region), - Some(expected_region.to_string()) + assert!( + builder + .get_config_value(&AmazonS3ConfigKey::Region) + .is_some() ); Ok(()) From bbd39f12f42883325ef8188322629b578674186d Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:37:25 +0000 Subject: [PATCH 08/16] prettier --- docs/source/contributor-guide/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md index 17c2b2d7de204..a69999a617faf 100644 --- a/docs/source/contributor-guide/index.md +++ b/docs/source/contributor-guide/index.md @@ -216,4 +216,4 @@ For those actions we also use the [Runs-On action](https://runs-on.com/caching/m For the standard GitHub runners, this action will do nothing. -We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. \ No newline at end of file +We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. From 2155e8ff477b8e161e4bf1d7f278212212b58dde Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:48:27 +0000 Subject: [PATCH 09/16] Extended tests --- .github/workflows/extended.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index e9eb27dd96527..a141eb3a522aa 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -66,7 +66,7 @@ jobs: # Check crate compiles and base cargo check passes linux-build-lib: name: linux build test - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -90,7 +90,7 @@ jobs: linux-test-extended: name: cargo test 'extended_tests' (amd64) needs: [linux-build-lib] - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -133,7 +133,7 @@ jobs: # Check answers are correct when hash values collide hash-collisions: name: cargo test hash collisions (amd64) - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: @@ -154,7 +154,7 @@ jobs: sqllogictest-sqlite: name: "Run sqllogictests with the sqlite test suite" - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: From acbfef75db554c1adfcb3cd9919b885d0c2cf65c Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 13:53:38 +0000 Subject: [PATCH 10/16] Extended tests --- .github/workflows/extended.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index a141eb3a522aa..594c02558e708 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -69,6 +69,7 @@ jobs: runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -93,6 +94,7 @@ jobs: runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -137,6 +139,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -158,6 +161,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push From 176f3430ea65c406f573f231ba6891ebf2051910 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 14:03:29 +0000 Subject: [PATCH 11/16] apt-get update? --- .github/workflows/extended.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index 594c02558e708..63c05ae0b0309 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -81,7 +81,9 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: sudo apt-get install -y protobuf-compiler + run: + sudo apt-get update + sudo apt-get install -y protobuf-compiler - name: Prepare cargo build run: | cargo check --profile ci --all-targets @@ -108,7 +110,9 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: sudo apt-get install -y protobuf-compiler + run: + sudo apt-get update + sudo apt-get install -y protobuf-compiler # For debugging, test binaries can be large. - name: Show available disk space run: | From 3611fde8253efda0ee2aa6049d4bf42d4f84e9ed Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 14:05:46 +0000 Subject: [PATCH 12/16] | --- .github/workflows/extended.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index 63c05ae0b0309..06f40c9a5c9ad 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -81,7 +81,7 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: + run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - name: Prepare cargo build @@ -110,7 +110,7 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: + run: | sudo apt-get update sudo apt-get install -y protobuf-compiler # For debugging, test binaries can be large. From c4a296ca800769b07a00b47a14944a0017f7f944 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 14:23:18 +0000 Subject: [PATCH 13/16] dependency cache --- .github/workflows/extended.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index 06f40c9a5c9ad..c8de028916916 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -175,6 +175,8 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Run sqllogictest run: | cargo test --features backtrace,parquet_encryption --profile release-nonlto --test sqllogictests -- --include-sqlite From f053effb55730f72610fcd22ef4dd7fb7003302e Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 14:47:51 +0000 Subject: [PATCH 14/16] let's do extended later --- .github/workflows/extended.yml | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index c8de028916916..e9eb27dd96527 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -66,10 +66,9 @@ jobs: # Check crate compiles and base cargo check passes linux-build-lib: name: linux build test - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ubuntu-latest # note: do not use amd/rust container to preserve disk space steps: - - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -81,9 +80,7 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler + run: sudo apt-get install -y protobuf-compiler - name: Prepare cargo build run: | cargo check --profile ci --all-targets @@ -93,10 +90,9 @@ jobs: linux-test-extended: name: cargo test 'extended_tests' (amd64) needs: [linux-build-lib] - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ubuntu-latest # note: do not use amd/rust container to preserve disk space steps: - - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -110,9 +106,7 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: | - sudo apt-get update - sudo apt-get install -y protobuf-compiler + run: sudo apt-get install -y protobuf-compiler # For debugging, test binaries can be large. - name: Show available disk space run: | @@ -139,11 +133,10 @@ jobs: # Check answers are correct when hash values collide hash-collisions: name: cargo test hash collisions (amd64) - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ubuntu-latest container: image: amd64/rust steps: - - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -161,11 +154,10 @@ jobs: sqllogictest-sqlite: name: "Run sqllogictests with the sqlite test suite" - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} + runs-on: ubuntu-latest container: image: amd64/rust steps: - - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -175,8 +167,6 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - name: Run sqllogictest run: | cargo test --features backtrace,parquet_encryption --profile release-nonlto --test sqllogictests -- --include-sqlite From 299fe344d529874a55bb5cfe2e5f76be72118a1e Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 15:08:32 +0000 Subject: [PATCH 15/16] Spot Instances note --- docs/source/contributor-guide/index.md | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md index a69999a617faf..5f7b789fa3b9d 100644 --- a/docs/source/contributor-guide/index.md +++ b/docs/source/contributor-guide/index.md @@ -202,13 +202,15 @@ It's recommended to write a high-quality issue with a clear problem statement an ### CI Runners -We use [Runs-On](https://runs-on.com/) for some actions in the main repository, which run in the ASF AWS account to speed up CI time. In forks, these actions run on the default GitHub runners since forks do not have access to ASF infrastructure. +#### Runs-On + +We use [Runs-On](https://runs-on.com/) for some actions in the main repository, which run in the ASF AWS account to speed up CI. In forks, these actions run on the default GitHub runners since forks do not have access to ASF infrastructure. To configure them, we use the following format: `runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}` -This is a conditional expression that uses Runs-On custom runners for the main repository but falls back to the standard GitHub runners for forks. Runs-On configuration follows the [Runs-On pattern](https://runs-on.com/configuration/job-labels/). +This is a conditional expression that uses Runs-On custom runners for the main repository and falls back to the standard GitHub runners for forks. Runs-On configuration follows the [Runs-On pattern](https://runs-on.com/configuration/job-labels/). For those actions we also use the [Runs-On action](https://runs-on.com/caching/magic-cache/#how-to-use), which adds support for external caching and reports job metrics: @@ -216,4 +218,15 @@ For those actions we also use the [Runs-On action](https://runs-on.com/caching/m For the standard GitHub runners, this action will do nothing. -We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. +##### Spot Instances + +By default, Runs-On actions run as [spot instances](https://runs-on.com/configuration/spot-instances/), which means they might occasionally be interrupted. In the CI you would see: +``` +Error: The operation was canceled. +``` + +According to Runs-On, spot instance termination is extremely rare for instances running for less than 1h. Those actions will be restarted automatically. + +#### GitHub Runners + +We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. \ No newline at end of file From a818b9212653b9dce9896e6bd0f620335d7804a6 Mon Sep 17 00:00:00 2001 From: blaginin Date: Sat, 21 Feb 2026 16:44:14 +0000 Subject: [PATCH 16/16] fmt --- docs/source/contributor-guide/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md index 5f7b789fa3b9d..2ee8a2aaac6cc 100644 --- a/docs/source/contributor-guide/index.md +++ b/docs/source/contributor-guide/index.md @@ -221,6 +221,7 @@ For the standard GitHub runners, this action will do nothing. ##### Spot Instances By default, Runs-On actions run as [spot instances](https://runs-on.com/configuration/spot-instances/), which means they might occasionally be interrupted. In the CI you would see: + ``` Error: The operation was canceled. ``` @@ -229,4 +230,4 @@ According to Runs-On, spot instance termination is extremely rare for instances #### GitHub Runners -We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks. \ No newline at end of file +We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks.