diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b0ae21f9801bf..0bcda10e3b6c3 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -185,7 +185,6 @@ jobs: fi - name: Check envs id: check-envs - if: inputs.branch != 'branch-3.5' env: ${{ fromJSON(inputs.envs) }} run: | if [[ "${{ fromJson(steps.set-outputs.outputs.required).pyspark }}" == 'true' || "${{ fromJson(steps.set-outputs.outputs.required).pyspark-pandas }}" == 'true' ]]; then @@ -253,17 +252,10 @@ jobs: run: | # Set the image URL for job "docs" # Should delete the link and directly use image_docs_url after SPARK 3.x EOL - if [[ "${{ inputs.branch }}" == 'branch-3.5' ]]; then - echo "image_docs_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT - echo "image_lint_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT - echo "image_sparkr_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT - echo "image_pyspark_url_link=${{ steps.infra-image-outputs.outputs.image_url }}" >> $GITHUB_OUTPUT - else - echo "image_docs_url_link=${{ steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT - echo "image_lint_url_link=${{ steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT - echo "image_sparkr_url_link=${{ steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT - echo "image_pyspark_url_link=${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >> $GITHUB_OUTPUT - fi + echo "image_docs_url_link=${{ steps.infra-image-docs-outputs.outputs.image_docs_url }}" >> $GITHUB_OUTPUT + echo "image_lint_url_link=${{ steps.infra-image-lint-outputs.outputs.image_lint_url }}" >> $GITHUB_OUTPUT + echo "image_sparkr_url_link=${{ steps.infra-image-sparkr-outputs.outputs.image_sparkr_url }}" >> $GITHUB_OUTPUT + echo "image_pyspark_url_link=${{ steps.infra-image-pyspark-outputs.outputs.image_pyspark_url }}" >> $GITHUB_OUTPUT # Build: build Spark and run the tests for specified modules. build: @@ -433,10 +425,6 @@ jobs: export TERM=vt100 # Hive "other tests" test needs larger metaspace size based on experiment. if [[ "$MODULES_TO_TEST" == "hive" ]] && [[ "$EXCLUDED_TAGS" == "org.apache.spark.tags.SlowHiveTest" ]]; then export METASPACE_SIZE=2g; fi - # SPARK-46283: should delete the following env replacement after SPARK 3.x EOL - if [[ "$MODULES_TO_TEST" == *"streaming-kinesis-asl"* ]] && [[ "${{ inputs.branch }}" =~ ^branch-3 ]]; then - MODULES_TO_TEST=${MODULES_TO_TEST//streaming-kinesis-asl, /} - fi if [ "${{ steps.extract-precompiled.outcome }}" = "success" ]; then export SKIP_SCALA_BUILD=true echo "Reusing precompiled artifact, skipping local SBT build." @@ -508,19 +496,8 @@ jobs: uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a - name: Set up Docker Buildx uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - - name: Build and push for branch-3.5 - if: inputs.branch == 'branch-3.5' - id: docker_build - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f - with: - context: ./dev/infra/ - push: true - tags: | - ${{ needs.precondition.outputs.image_url }} - # Use the infra image cache to speed up - cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-cache:${{ inputs.branch }} - name: Build and push (Documentation) - if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).docs == 'true' && hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }} + if: ${{ fromJson(needs.precondition.outputs.required).docs == 'true' && hashFiles('dev/spark-test-image/docs/Dockerfile') != '' }} id: docker_build_docs uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: @@ -531,7 +508,7 @@ jobs: # Use the infra image cache to speed up cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-docs-cache:${{ inputs.branch }} - name: Build and push (Linter) - if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).lint == 'true' && hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }} + if: ${{ fromJson(needs.precondition.outputs.required).lint == 'true' && hashFiles('dev/spark-test-image/lint/Dockerfile') != '' }} id: docker_build_lint uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: @@ -542,7 +519,7 @@ jobs: # Use the infra image cache to speed up cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-lint-cache:${{ inputs.branch }} - name: Build and push (SparkR) - if: ${{ inputs.branch != 'branch-3.5' && fromJson(needs.precondition.outputs.required).sparkr == 'true' && hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }} + if: ${{ fromJson(needs.precondition.outputs.required).sparkr == 'true' && hashFiles('dev/spark-test-image/sparkr/Dockerfile') != '' }} id: docker_build_sparkr uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f with: @@ -553,7 +530,7 @@ jobs: # Use the infra image cache to speed up cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-sparkr-cache:${{ inputs.branch }} - name: Build and push (PySpark with ${{ env.PYSPARK_IMAGE_TO_TEST }}) - if: ${{ inputs.branch != 'branch-3.5' && (fromJson(needs.precondition.outputs.required).pyspark == 'true' || fromJson(needs.precondition.outputs.required).pyspark-pandas == 'true') && env.PYSPARK_IMAGE_TO_TEST != '' }} + if: ${{ (fromJson(needs.precondition.outputs.required).pyspark == 'true' || fromJson(needs.precondition.outputs.required).pyspark-pandas == 'true') && env.PYSPARK_IMAGE_TO_TEST != '' }} id: docker_build_pyspark env: ${{ fromJSON(inputs.envs) }} uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f @@ -796,12 +773,7 @@ jobs: export SKIP_PACKAGING=false echo "Python Packaging Tests Enabled!" fi - if [ ! -z "$PYTHON_TO_TEST" ]; then - ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --python-executables "$PYTHON_TO_TEST" - else - # For branch-3.5 and below, it uses the default Python versions. - ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" - fi + ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" --python-executables "$PYTHON_TO_TEST" - name: Run tests for old client env: SPARK_TESTING: 1 @@ -1018,21 +990,11 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.12' - - name: Install dependencies for Python CodeGen check (branch-3.5, branch-4.0, branch-4.1) - if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0' || inputs.branch == 'branch-4.1' - run: | - python3.12 -m pip install 'black==23.12.1' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0' - python3.12 -m pip list - name: Install dependencies for Python CodeGen check - if: ${{ inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1' }} run: | python3.12 -m pip install 'ruff==0.14.8' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0' python3.12 -m pip list - - name: Python CodeGen check for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: ./dev/connect-check-protos.py - name: Python CodeGen check - if: inputs.branch != 'branch-3.5' run: ./dev/check-protos.py # Static analysis @@ -1138,87 +1100,23 @@ jobs: if: hashFiles('dev/structured_logging_style.py') != '' shell: 'script -q -e -c "bash {0}"' run: | - if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then - python3.9 ./dev/structured_logging_style.py - elif [[ "$BRANCH" == 'branch-4.2' || "$BRANCH" == 'branch-4.x' ]]; then - python3.12 ./dev/structured_logging_style.py - elif [[ "$BRANCH" == 'branch-4.1' ]]; then - python3.11 ./dev/structured_logging_style.py - else - python3.12 ./dev/structured_logging_style.py - fi + python3.12 ./dev/structured_logging_style.py - name: Java linter run: ./dev/lint-java - name: Spark connect jvm client mima check run: ./dev/connect-jvm-client-mima-check - - name: Install Python linter dependencies for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: | - # SPARK-45212: Copy from https://github.com/apache/spark/blob/555c8def51e5951c7bf5165a332795e9e330ec9d/.github/workflows/build_and_test.yml#L631-L638 - # Should delete this section after SPARK 3.5 EOL. - python3.9 -m pip install 'flake8==3.9.0' pydata_sphinx_theme 'mypy==0.982' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' numpydoc 'jinja2<3.0.0' 'black==22.6.0' - python3.9 -m pip install 'pandas-stubs==1.2.0.53' ipython 'grpcio==1.56.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' - name: List Python packages shell: 'script -q -e -c "bash {0}"' run: | lsb_release -a - if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then - python3.9 --version - python3.9 -m pip list - elif [[ "$BRANCH" == 'branch-4.2' || "$BRANCH" == 'branch-4.x' ]]; then - python3.12 --version - python3.12 -m pip list - elif [[ "$BRANCH" == 'branch-4.1' ]]; then - python3.11 --version - python3.11 -m pip list - else - python3.12 --version - python3.12 -m pip list - fi + python3.12 --version + python3.12 -m pip list - name: Python linter shell: 'script -q -e -c "bash {0}"' run: | - if [[ "$BRANCH" == 'branch-3.5' || "$BRANCH" == 'branch-4.0' ]]; then - PYTHON_EXECUTABLE=python3.9 ./dev/lint-python - elif [[ "$BRANCH" == 'branch-4.2' || "$BRANCH" == 'branch-4.x' ]]; then - PYTHON_EXECUTABLE=python3.12 ./dev/lint-python - elif [[ "$BRANCH" == 'branch-4.1' ]]; then - PYTHON_EXECUTABLE=python3.11 ./dev/lint-python - else - PYTHON_EXECUTABLE=python3.12 ./dev/lint-python - fi - # Should delete this section after SPARK 3.5 EOL. - - name: Install dependencies for Python code generation check for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: | - # See more in "Installation" https://docs.buf.build/installation#tarball - curl -LO https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64.tar.gz - mkdir -p $HOME/buf - tar -xvzf buf-Linux-x86_64.tar.gz -C $HOME/buf --strip-components 1 - rm buf-Linux-x86_64.tar.gz - python3.9 -m pip install 'protobuf==4.25.1' 'mypy-protobuf==3.3.0' - # Should delete this section after SPARK 3.5 EOL. - - name: Python code generation check for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: if test -f ./dev/connect-check-protos.py; then PATH=$PATH:$HOME/buf/bin PYTHON_EXECUTABLE=python3.9 ./dev/connect-check-protos.py; fi - # Should delete this section after SPARK 3.5 EOL. - - name: Install JavaScript linter dependencies for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: | - apt update - apt-get install -y nodejs npm + PYTHON_EXECUTABLE=python3.12 ./dev/lint-python - name: JS linter run: ./dev/lint-js - # Should delete this section after SPARK 3.5 EOL. - - name: Install R linter dependencies for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: | - apt update - apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev \ - libfontconfig1-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev libpng-dev \ - libtiff5-dev libjpeg-dev - Rscript -e "install.packages(c('remotes'), repos='https://cloud.r-project.org/')" - Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" - name: Install R linter dependencies and SparkR run: ./R/install-dev.sh - name: R linter @@ -1307,32 +1205,7 @@ jobs: with: distribution: zulu java-version: ${{ inputs.java }} - - name: Install dependencies for documentation generation for branch-3.5 - if: inputs.branch == 'branch-3.5' - run: | - # pandoc is required to generate PySpark APIs as well in nbsphinx. - apt-get update -y - apt-get install -y libcurl4-openssl-dev pandoc - apt-get install -y ruby ruby-dev - Rscript -e "install.packages(c('remotes', 'testthat', 'knitr', 'rmarkdown', 'markdown', 'e1071', 'roxygen2', 'ggplot2', 'mvtnorm', 'statmod'), repos='https://cloud.r-project.org/')" - Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" - Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" - # Should unpin 'sphinxcontrib-*' after upgrading sphinx>5 - python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' - python3.9 -m pip install ipython_genutils # See SPARK-38517 - python3.9 -m pip install sphinx_plotly_directive 'numpy>=1.22' pyarrow pandas 'plotly<6.0.0' - python3.9 -m pip install 'docutils<0.18.0' # See SPARK-39421 - - name: List Python packages for branch-3.5 and branch-4.0 - if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0' - run: python3.9 -m pip list - - name: List Python packages for branch-4.2 and branch-4.x (Python 3.12) - if: inputs.branch == 'branch-4.2' || inputs.branch == 'branch-4.x' - run: python3.12 -m pip list - - name: List Python packages for branch-4.1 - if: inputs.branch == 'branch-4.1' - run: python3.11 -m pip list - name: List Python packages - if: ${{ inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1' && inputs.branch != 'branch-4.2' && inputs.branch != 'branch-4.x' }} run: | lsb_release -a python3.12 -m pip list @@ -1344,78 +1217,7 @@ jobs: gem install bundler -v 2.4.22 cd docs bundle install --retry=100 - - name: Run documentation build for branch-3.5 and branch-4.0 - if: inputs.branch == 'branch-3.5' || inputs.branch == 'branch-4.0' - run: | - # We need this link to make sure `python3` points to `python3.9` which contains the prerequisite packages. - ln -s "$(which python3.9)" "/usr/local/bin/python3" - # Build docs first with SKIP_API to ensure they are buildable without requiring any - # language docs to be built beforehand. - cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd .. - if [ -f "./dev/is-changed.py" ]; then - # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs - pyspark_modules=`cd dev && python3.9 -c "import sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if m.name.startswith('pyspark')))"` - if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then export SKIP_PYTHONDOC=1; fi - if [ `./dev/is-changed.py -m sparkr` = false ]; then export SKIP_RDOC=1; fi - fi - # Print the values of environment variables `SKIP_ERRORDOC`, `SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC` - echo "SKIP_ERRORDOC: $SKIP_ERRORDOC" - echo "SKIP_SCALADOC: $SKIP_SCALADOC" - echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC" - echo "SKIP_RDOC: $SKIP_RDOC" - echo "SKIP_SQLDOC: $SKIP_SQLDOC" - cd docs - bundle exec jekyll build - - name: Run documentation build for branch-4.2 and branch-4.x (Python 3.12) - if: inputs.branch == 'branch-4.2' || inputs.branch == 'branch-4.x' - run: | - # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages. - ln -s "$(which python3.12)" "/usr/local/bin/python3" - # Build docs first with SKIP_API to ensure they are buildable without requiring any - # language docs to be built beforehand. - cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd .. - if [ -f "./dev/is-changed.py" ]; then - # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs - pyspark_modules=`cd dev && python3.12 -c "import sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if m.name.startswith('pyspark')))"` - if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then export SKIP_PYTHONDOC=1; fi - if [ `./dev/is-changed.py -m sparkr` = false ]; then export SKIP_RDOC=1; fi - fi - export PYSPARK_DRIVER_PYTHON=python3.12 - export PYSPARK_PYTHON=python3.12 - # Print the values of environment variables `SKIP_ERRORDOC`, `SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC` - echo "SKIP_ERRORDOC: $SKIP_ERRORDOC" - echo "SKIP_SCALADOC: $SKIP_SCALADOC" - echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC" - echo "SKIP_RDOC: $SKIP_RDOC" - echo "SKIP_SQLDOC: $SKIP_SQLDOC" - cd docs - bundle exec jekyll build - - name: Run documentation build for branch-4.1 - if: inputs.branch == 'branch-4.1' - run: | - # We need this link to make sure `python3` points to `python3.11` which contains the prerequisite packages. - ln -s "$(which python3.11)" "/usr/local/bin/python3" - # Build docs first with SKIP_API to ensure they are buildable without requiring any - # language docs to be built beforehand. - cd docs; SKIP_ERRORDOC=1 SKIP_API=1 bundle exec jekyll build; cd .. - if [ -f "./dev/is-changed.py" ]; then - # Skip PySpark and SparkR docs while keeping Scala/Java/SQL docs - pyspark_modules=`cd dev && python3.11 -c "import sparktestsupport.modules as m; print(','.join(m.name for m in m.all_modules if m.name.startswith('pyspark')))"` - if [ `./dev/is-changed.py -m $pyspark_modules` = false ]; then export SKIP_PYTHONDOC=1; fi - if [ `./dev/is-changed.py -m sparkr` = false ]; then export SKIP_RDOC=1; fi - fi - export PYSPARK_DRIVER_PYTHON=python3.11 - export PYSPARK_PYTHON=python3.11 - # Print the values of environment variables `SKIP_ERRORDOC`, `SKIP_SCALADOC`, `SKIP_PYTHONDOC`, `SKIP_RDOC` and `SKIP_SQLDOC` - echo "SKIP_ERRORDOC: $SKIP_ERRORDOC" - echo "SKIP_SCALADOC: $SKIP_SCALADOC" - echo "SKIP_PYTHONDOC: $SKIP_PYTHONDOC" - echo "SKIP_RDOC: $SKIP_RDOC" - echo "SKIP_SQLDOC: $SKIP_SQLDOC" - cd docs - bundle exec jekyll build - name: Run documentation build - if: ${{ inputs.branch != 'branch-3.5' && inputs.branch != 'branch-4.0' && inputs.branch != 'branch-4.1' && inputs.branch != 'branch-4.2' && inputs.branch != 'branch-4.x' }} run: | # We need this link to make sure `python3` points to `python3.12` which contains the prerequisite packages. ln -s "$(which python3.12)" "/usr/local/bin/python3" @@ -1550,7 +1352,6 @@ jobs: spark.sql.autoBroadcastJoinThreshold=-1 spark.sql.join.forceApplyShuffledHashJoin=true - name: Run TPC-DS queries on collated data - if: inputs.branch != 'branch-3.5' run: | SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 build/sbt "sql/testOnly org.apache.spark.sql.TPCDSCollationQueryTestSuite" - name: Upload test results to report @@ -1754,13 +1555,7 @@ jobs: export PVC_TESTS_VM_PATH=$PVC_TMP_DIR minikube mount ${PVC_TESTS_HOST_PATH}:${PVC_TESTS_VM_PATH} --gid=0 --uid=185 & kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true - if [[ "${{ inputs.branch }}" == 'branch-3.5' ]]; then - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.7.0/installer/volcano-development.yaml || true - elif [[ "${{ inputs.branch }}" == 'branch-4.0' ]]; then - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.11.0/installer/volcano-development.yaml || true - else - kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.14.2/installer/volcano-development.yaml || true - fi + kubectl apply -f https://raw.githubusercontent.com/volcano-sh/volcano/v1.14.2/installer/volcano-development.yaml || true eval $(minikube docker-env) build/sbt -Phadoop-3 -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test" - name: Upload Spark on K8S integration tests log files