From 81cddbb77c2e555c044956d57e0b393f59c95ecc Mon Sep 17 00:00:00 2001 From: Lin Liu <141371752+linliu-code@users.noreply.github.com> Date: Sat, 10 Feb 2024 11:33:03 -0800 Subject: [PATCH] [HUDI-6902] Containerize the Azure CI (#10512) * [HUDI-6902] Containerize the Azure tests * remove warning message --- .github/workflows/bot.yml | 6 +- Dockerfile | 31 +++ azure-pipelines-20230430.yml | 190 +++++++++--------- .../hudi-metaserver-server/pom.xml | 49 +++-- pom.xml | 1 + 5 files changed, 154 insertions(+), 123 deletions(-) create mode 100644 Dockerfile diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index a31c2e3ea35c..6c80b621cbcd 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -150,7 +150,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} FLINK_PROFILE: ${{ matrix.flinkProfile }} run: - ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client + ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service -Pthrift-gen-source-with-script $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -335,10 +335,10 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SCALA_PROFILE: 'scala-2.12' run: | - mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS + mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -Phudi-platform-service -Pthrift-gen-source-with-script # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708. sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources - mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 + mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 -Phudi-platform-service -Pthrift-gen-source-with-script - name: IT - Bundle Validation - OpenJDK 8 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000000..f8d038771435 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use a home made image as the base, which includes: +# utuntu:latest +# git +# thrift +# maven +# java8 +# Use an official Ubuntu base image +FROM apachehudi/hudi-ci-bundle-validation-base:azure_ci_test_base_new + +CMD ["java", "-version"] + +# Set the working directory to /app +WORKDIR /hudi + +# Copy git repo into the working directory +COPY . /hudi \ No newline at end of file diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml index 0767d179b243..559686a2292f 100644 --- a/azure-pipelines-20230430.yml +++ b/azure-pipelines-20230430.yml @@ -98,7 +98,7 @@ parameters: variables: BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18' PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn' - MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5' + MVN_OPTS_INSTALL: '-DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5' MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)' JOB1_MODULES: ${{ join(',',parameters.job1Modules) }} JOB2_MODULES: ${{ join(',',parameters.job2Modules) }} @@ -108,128 +108,120 @@ variables: stages: - stage: test + variables: + - name: DOCKER_BUILDKIT + value: 1 jobs: - job: UT_FT_1 displayName: UT FT common & flink & UT client/spark-client timeoutInMinutes: '150' steps: - - task: Maven@4 - displayName: maven install + - task: Docker@2 + displayName: "login to docker" inputs: - mavenPomFile: 'pom.xml' - goals: 'clean install' - options: $(MVN_OPTS_INSTALL) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - - task: Maven@4 - displayName: UT common flink client/spark-client + command: "login" + containerRegistry: "apachehudi-docker-hub" + - task: Docker@2 + displayName: "load repo into image" inputs: - mavenPomFile: 'pom.xml' - goals: 'test' - options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - mavenOptions: '-Xmx4g' - - task: Maven@4 - displayName: FT common flink + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'build' + Dockerfile: '**/Dockerfile' + ImageName: $(Build.BuildId) + - task: Docker@2 + displayName: "UT FT common flink client/spark-client" inputs: - mavenPomFile: 'pom.xml' - goals: 'test' - options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - mavenOptions: '-Xmx4g' - - script: | - grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 - displayName: Top 100 long-running testcases + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'run' + arguments: > + -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) + /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) + && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client + && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES) + && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" - job: UT_FT_2 displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark timeoutInMinutes: '150' steps: - - task: Maven@4 - displayName: maven install + - task: Docker@2 + displayName: "login to docker" inputs: - mavenPomFile: 'pom.xml' - goals: 'clean install' - options: $(MVN_OPTS_INSTALL) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - - task: Maven@4 - displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark + command: "login" + containerRegistry: "apachehudi-docker-hub" + - task: Docker@2 + displayName: "load repo into image" inputs: - mavenPomFile: 'pom.xml' - goals: 'test' - options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - mavenOptions: '-Xmx4g' - - script: | - grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 - displayName: Top 100 long-running testcases + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'build' + Dockerfile: '**/Dockerfile' + ImageName: $(Build.BuildId) + - task: Docker@2 + displayName: "FT client/spark-client & hudi-spark-datasource/hudi-spark" + inputs: + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'run' + arguments: > + -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) + /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) + && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES) + && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" - job: UT_FT_3 displayName: UT spark-datasource timeoutInMinutes: '240' steps: - - task: Maven@4 - displayName: maven install + - task: Docker@2 + displayName: "login to docker" + inputs: + command: "login" + containerRegistry: "apachehudi-docker-hub" + - task: Docker@2 + displayName: "load repo into image" inputs: - mavenPomFile: 'pom.xml' - goals: 'clean install' - options: $(MVN_OPTS_INSTALL) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - - task: Maven@4 - displayName: UT spark-datasource + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'build' + Dockerfile: '**/Dockerfile' + ImageName: $(Build.BuildId) + - task: Docker@2 + displayName: "UT spark-datasource" inputs: - mavenPomFile: 'pom.xml' - goals: 'test' - options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - mavenOptions: '-Xmx4g' - - script: | - grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 - displayName: Top 100 long-running testcases + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'run' + arguments: > + -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) + /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) + && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" - job: UT_FT_4 displayName: UT FT other modules timeoutInMinutes: '240' steps: - - task: Maven@4 - displayName: maven install + - task: Docker@2 + displayName: "login to docker hub" inputs: - mavenPomFile: 'pom.xml' - goals: 'clean install' - options: $(MVN_OPTS_INSTALL) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - - task: Maven@4 - displayName: UT other modules + command: "login" + containerRegistry: "apachehudi-docker-hub" + - task: Docker@2 + displayName: "load repo into image" inputs: - mavenPomFile: 'pom.xml' - goals: 'test' - options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - mavenOptions: '-Xmx4g' - - task: Maven@4 - displayName: FT other modules + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'build' + Dockerfile: '**/Dockerfile' + ImageName: $(Build.BuildId) + - task: Docker@2 + displayName: "UT FT other modules" inputs: - mavenPomFile: 'pom.xml' - goals: 'test' - options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES) - publishJUnitResults: true - testResultsFiles: '**/surefire-reports/TEST-*.xml' - jdkVersionOption: '1.8' - mavenOptions: '-Xmx4g' - - script: | - grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 - displayName: Top 100 long-running testcases + containerRegistry: 'apachehudi-docker-hub' + repository: 'apachehudi/hudi-ci-bundle-validation-base' + command: 'run' + arguments: > + -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId) + /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source + && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES) + && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_UT_MODULES) + && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100" \ No newline at end of file diff --git a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml index 4ffbea73b44a..6b6f67a1e382 100644 --- a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml +++ b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml @@ -92,6 +92,34 @@ + + thrift-gen-source-with-script + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + thrift-install-and-generate-source + generate-sources + + exec + + + + + ${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh + + ${thrift.install.env} + + false + + + + + m1-mac @@ -108,27 +136,6 @@ - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - thrift-install-and-generate-source - generate-sources - - exec - - - - - ${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh - - ${thrift.install.env} - - false - - org.jacoco jacoco-maven-plugin diff --git a/pom.xml b/pom.xml index 3eeed340178b..df14c011e45a 100644 --- a/pom.xml +++ b/pom.xml @@ -232,6 +232,7 @@ 2.7.3 2.1.1 1.1.8.3 + /usr/local/bin/thrift