From 81cddbb77c2e555c044956d57e0b393f59c95ecc Mon Sep 17 00:00:00 2001
From: Lin Liu <141371752+linliu-code@users.noreply.github.com>
Date: Sat, 10 Feb 2024 11:33:03 -0800
Subject: [PATCH] [HUDI-6902] Containerize the Azure CI (#10512)
* [HUDI-6902] Containerize the Azure tests
* remove warning message
---
.github/workflows/bot.yml | 6 +-
Dockerfile | 31 +++
azure-pipelines-20230430.yml | 190 +++++++++---------
.../hudi-metaserver-server/pom.xml | 49 +++--
pom.xml | 1 +
5 files changed, 154 insertions(+), 123 deletions(-)
create mode 100644 Dockerfile
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index a31c2e3ea35c..6c80b621cbcd 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -150,7 +150,7 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
FLINK_PROFILE: ${{ matrix.flinkProfile }}
run:
- ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
+ ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service -Pthrift-gen-source-with-script $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
- name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client
env:
SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -335,10 +335,10 @@ jobs:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
SCALA_PROFILE: 'scala-2.12'
run: |
- mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS
+ mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -Phudi-platform-service -Pthrift-gen-source-with-script
# TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708.
sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources
- mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0
+ mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 -Phudi-platform-service -Pthrift-gen-source-with-script
- name: IT - Bundle Validation - OpenJDK 8
env:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 000000000000..f8d038771435
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use a home made image as the base, which includes:
+# utuntu:latest
+# git
+# thrift
+# maven
+# java8
+# Use an official Ubuntu base image
+FROM apachehudi/hudi-ci-bundle-validation-base:azure_ci_test_base_new
+
+CMD ["java", "-version"]
+
+# Set the working directory to /app
+WORKDIR /hudi
+
+# Copy git repo into the working directory
+COPY . /hudi
\ No newline at end of file
diff --git a/azure-pipelines-20230430.yml b/azure-pipelines-20230430.yml
index 0767d179b243..559686a2292f 100644
--- a/azure-pipelines-20230430.yml
+++ b/azure-pipelines-20230430.yml
@@ -98,7 +98,7 @@ parameters:
variables:
BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18'
PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
- MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
+ MVN_OPTS_INSTALL: '-DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
JOB2_MODULES: ${{ join(',',parameters.job2Modules) }}
@@ -108,128 +108,120 @@ variables:
stages:
- stage: test
+ variables:
+ - name: DOCKER_BUILDKIT
+ value: 1
jobs:
- job: UT_FT_1
displayName: UT FT common & flink & UT client/spark-client
timeoutInMinutes: '150'
steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: UT common flink client/spark-client
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - task: Maven@4
- displayName: FT common flink
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "UT FT common flink client/spark-client"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
- job: UT_FT_2
displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark
timeoutInMinutes: '150'
steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "FT client/spark-client & hudi-spark-datasource/hudi-spark"
+ inputs:
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
- job: UT_FT_3
displayName: UT spark-datasource
timeoutInMinutes: '240'
steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker"
+ inputs:
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: UT spark-datasource
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "UT spark-datasource"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
- job: UT_FT_4
displayName: UT FT other modules
timeoutInMinutes: '240'
steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker hub"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: UT other modules
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - task: Maven@4
- displayName: FT other modules
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "UT FT other modules"
inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_UT_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
\ No newline at end of file
diff --git a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml
index 4ffbea73b44a..6b6f67a1e382 100644
--- a/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml
+++ b/hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml
@@ -92,6 +92,34 @@
+
+ thrift-gen-source-with-script
+
+
+
+ org.codehaus.mojo
+ exec-maven-plugin
+ 1.6.0
+
+
+ thrift-install-and-generate-source
+ generate-sources
+
+ exec
+
+
+
+
+ ${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh
+
+ ${thrift.install.env}
+
+ false
+
+
+
+
+
m1-mac
@@ -108,27 +136,6 @@
-
- org.codehaus.mojo
- exec-maven-plugin
- 1.6.0
-
-
- thrift-install-and-generate-source
- generate-sources
-
- exec
-
-
-
-
- ${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh
-
- ${thrift.install.env}
-
- false
-
-
org.jacoco
jacoco-maven-plugin
diff --git a/pom.xml b/pom.xml
index 3eeed340178b..df14c011e45a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -232,6 +232,7 @@
2.7.3
2.1.1
1.1.8.3
+ /usr/local/bin/thrift