diff --git a/.asf.yaml b/.asf.yaml index 34d036e19b409..f235c6dd9704a 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -68,6 +68,8 @@ github: - "cargo check datafusion-functions features" - "cargo check datafusion-spark features" - "cargo test (amd64)" + - "cargo test 'extended_tests' (amd64)" + - "cargo test hash collisions (amd64)" - "cargo test datafusion-cli (amd64)" - "cargo examples (amd64)" - "cargo test doc (amd64)" @@ -76,6 +78,7 @@ github: - "verify benchmark results (amd64)" - "Run sqllogictest with Postgres runner" - "Run sqllogictest in Substrait round-trip mode" + - "Run sqllogictests with the sqlite test suite" - "cargo test (macos-aarch64)" - "Verify Vendored Code" - "Check cargo fmt" diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index d2f1f711754ab..cdddc3c9a4c4b 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -21,31 +21,18 @@ concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true -# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges -# -# These jobs are not run as part of PR checks as they are time-consuming -# and should not fail often. -# -# We still run them as they provide important coverage to ensure correctness -# in the (very rare) event of a hash failure or sqlite library query failure. +# Run these required checks in the merge queue before merging into main. +# On PR updates, skip the jobs so their required checks allow queue entry +# without running the expensive suites. Do not add pull_request path filters: +# a skipped workflow would leave required checks pending. on: + pull_request: + merge_group: push: branches: - - main - # support extended test suite for release candidate branches, - # it is not expected to have many changes in these branches, - # so running extended tests is not a burden + # Release branches retain post-push coverage. main is covered by the + # merge queue, so do not repeat the tests after merging. - 'branch-*' - # Also run for changes to some critical areas that are most likely - # to trigger errors in extended tests - pull_request: - branches: [ '**' ] - paths: - - 'datafusion/physical*/**/*.rs' - - 'datafusion/expr*/**/*.rs' - - 'datafusion/optimizer/**/*.rs' - - 'datafusion/sql/**/*.rs' - - 'datafusion-testing' workflow_dispatch: inputs: pr_head_sha: @@ -60,13 +47,15 @@ jobs: # Run extended tests (with feature 'extended_tests') linux-test-extended: name: cargo test 'extended_tests' (amd64) + # Long-running job: run in the merge queue, on release pushes, or manually. + if: github.event_name != 'pull_request' runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push + ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group submodules: true fetch-depth: 1 - name: Free Disk Space (Ubuntu) @@ -111,6 +100,8 @@ jobs: # Check answers are correct when hash values collide hash-collisions: name: cargo test hash collisions (amd64) + # Long-running job: run in the merge queue, on release pushes, or manually. + if: github.event_name != 'pull_request' runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -118,7 +109,7 @@ jobs: - uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push + ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group submodules: true fetch-depth: 1 - name: Setup Rust toolchain @@ -133,6 +124,8 @@ jobs: sqllogictest-sqlite: name: "Run sqllogictests with the sqlite test suite" + # Long-running job: run in the merge queue, on release pushes, or manually. + if: github.event_name != 'pull_request' runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust @@ -141,7 +134,7 @@ jobs: - parallel: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push + ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group submodules: true fetch-depth: 1 # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md index c344d67159f33..44bd327013a49 100644 --- a/docs/source/contributor-guide/testing.md +++ b/docs/source/contributor-guide/testing.md @@ -111,7 +111,7 @@ cargo test --profile=ci --test sqllogictests -- --complete Like similar systems such as [DuckDB](https://duckdb.org/dev/testing), DataFusion has chosen to trade off a slightly higher barrier to contribution for longer term maintainability. -DataFusion has integrated [sqlite's test suite](https://sqlite.org/sqllogictest/doc/trunk/about.wiki) as a supplemental test suite that is run whenever a PR is merged into DataFusion. To run it manually please refer to the [README](https://github.com/apache/datafusion/blob/main/datafusion/sqllogictest/README.md#running-tests-sqlite) file for instructions. +DataFusion runs [sqlite's test suite](https://sqlite.org/sqllogictest/doc/trunk/about.wiki) in the merge queue before merging PRs into `main`. For local instructions, see [Running Tests: sqlite](https://github.com/apache/datafusion/blob/main/datafusion/sqllogictest/README.md#running-tests-sqlite). ## Snapshot testing (`cargo insta`) @@ -127,15 +127,15 @@ cargo insta review ## Extended Tests -In addition to the standard CI test suite that is run on all PRs prior to merge, -DataFusion has "extended" tests (defined in [extended.yml]) that are run on each -commit to `main`. These tests rarely fail but take significantly longer to run -than the standard test suite and add important test coverage such as ensuring -correctness when there are hash collisions and running the relevant portions of -the entire [sqlite test suite]. You can run the extended tests -locally by following the [instructions in the documentation]. +DataFusion has extended tests (defined in [extended.yml]) that take significantly +longer to run than the standard suite. They provide additional correctness +coverage and must pass in the merge queue before a PR merges into `main`. + +To conserve CI resources, these tests do not run on ordinary PR updates. They +also run on pushes to release branches (`branch-*`). You can run them manually. + +For local SQLite test instructions, see the [instructions in the documentation]. -[sqlite test suite]: https://www.sqlite.org/sqllogictest/dir?ci=tip [instructions in the documentation]: https://github.com/apache/datafusion/tree/main/datafusion/sqllogictest#running-tests-sqlite [extended.yml]: https://github.com/apache/datafusion/blob/main/.github/workflows/extended.yml