[HUDI-6902] Containerize the Azure CI (apache#10512)

* [HUDI-6902] Containerize the Azure tests

* remove warning message

.github/workflows/bot.yml

@@ -150,7 +150,7 @@ jobs:
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  FLINK_PROFILE: ${{ matrix.flinkProfile }}
  run:
- ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
+ ./mvnw clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"FLINK_PROFILE" -DskipTests=true -Phudi-platform-service -Pthrift-gen-source-with-script $MVN_ARGS -am -pl hudi-hadoop-mr,hudi-client/hudi-java-client
  - name: UT - hudi-hadoop-mr and hudi-client/hudi-java-client
  env:
  SCALA_PROFILE: ${{ matrix.scalaProfile }}
@@ -335,10 +335,10 @@ jobs:
  SPARK_PROFILE: ${{ matrix.sparkProfile }}
  SCALA_PROFILE: 'scala-2.12'
  run: |
- mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS
+ mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -Phudi-platform-service -Pthrift-gen-source-with-script
  # TODO remove the sudo below. It's a needed workaround as detailed in HUDI-5708.
  sudo chown -R "$USER:$(id -g -n)" hudi-platform-service/hudi-metaserver/target/generated-sources
- mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0
+ mvn clean package -T 2 -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -DdeployArtifacts=true -DskipTests=true $MVN_ARGS -pl packaging/hudi-flink-bundle -am -Davro.version=1.10.0 -Phudi-platform-service -Pthrift-gen-source-with-script
  - name: IT - Bundle Validation - OpenJDK 8
  env:
  FLINK_PROFILE: ${{ matrix.flinkProfile }}

Dockerfile

@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use a home made image as the base, which includes:
+# utuntu:latest
+# git
+# thrift
+# maven
+# java8
+# Use an official Ubuntu base image
+FROM apachehudi/hudi-ci-bundle-validation-base:azure_ci_test_base_new
+
+CMD ["java", "-version"]
+
+# Set the working directory to /app
+WORKDIR /hudi
+
+# Copy git repo into the working directory
+COPY . /hudi

azure-pipelines-20230430.yml

@@ -98,7 +98,7 @@ parameters:
 variables:
  BUILD_PROFILES: '-Dscala-2.12 -Dspark3.2 -Dflink1.18'
  PLUGIN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true -ntp -B -V -Pwarn-log -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.shade=warn -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugins.dependency=warn'
- MVN_OPTS_INSTALL: '-Phudi-platform-service -DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
+ MVN_OPTS_INSTALL: '-DskipTests $(BUILD_PROFILES) $(PLUGIN_OPTS) -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=5'
  MVN_OPTS_TEST: '-fae -Pwarn-log $(BUILD_PROFILES) $(PLUGIN_OPTS)'
  JOB1_MODULES: ${{ join(',',parameters.job1Modules) }}
  JOB2_MODULES: ${{ join(',',parameters.job2Modules) }}
@@ -108,128 +108,120 @@ variables:
 
 stages:
  - stage: test
+ variables:
+ - name: DOCKER_BUILDKIT
+ value: 1
  jobs:
  - job: UT_FT_1
  displayName: UT FT common & flink & UT client/spark-client
  timeoutInMinutes: '150'
  steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: UT common flink client/spark-client
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - task: Maven@4
- displayName: FT common flink
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "UT FT common flink client/spark-client"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
  - job: UT_FT_2
  displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark
  timeoutInMinutes: '150'
  steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: FT client/spark-client & hudi-spark-datasource/hudi-spark
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "FT client/spark-client & hudi-spark-datasource/hudi-spark"
+ inputs:
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
  - job: UT_FT_3
  displayName: UT spark-datasource
  timeoutInMinutes: '240'
  steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker"
+ inputs:
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: UT spark-datasource
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "UT spark-datasource"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"
  - job: UT_FT_4
  displayName: UT FT other modules
  timeoutInMinutes: '240'
  steps:
- - task: Maven@4
- displayName: maven install
+ - task: Docker@2
+ displayName: "login to docker hub"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'clean install'
- options: $(MVN_OPTS_INSTALL)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- - task: Maven@4
- displayName: UT other modules
+ command: "login"
+ containerRegistry: "apachehudi-docker-hub"
+ - task: Docker@2
+ displayName: "load repo into image"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - task: Maven@4
- displayName: FT other modules
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'build'
+ Dockerfile: '**/Dockerfile'
+ ImageName: $(Build.BuildId)
+ - task: Docker@2
+ displayName: "UT FT other modules"
  inputs:
- mavenPomFile: 'pom.xml'
- goals: 'test'
- options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_FT_MODULES)
- publishJUnitResults: true
- testResultsFiles: '**/surefire-reports/TEST-*.xml'
- jdkVersionOption: '1.8'
- mavenOptions: '-Xmx4g'
- - script: |
- grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
- displayName: Top 100 long-running testcases
+ containerRegistry: 'apachehudi-docker-hub'
+ repository: 'apachehudi/hudi-ci-bundle-validation-base'
+ command: 'run'
+ arguments: >
+ -i docker.io/apachehudi/hudi-ci-bundle-validation-base:$(Build.BuildId)
+ /bin/bash -c "mvn clean install $(MVN_OPTS_INSTALL) -Phudi-platform-service -Pthrift-gen-source
+ && mvn test $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB4_UT_MODULES)
+ && mvn test $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_UT_MODULES)
+ && grep \"testcase\" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'\"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100"

hudi-platform-service/hudi-metaserver/hudi-metaserver-server/pom.xml

@@ -92,6 +92,34 @@
  </plugins>
  </build>
  </profile>
+ <profile>
+ <id>thrift-gen-source-with-script</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
+ <executions>
+ <execution>
+ <id>thrift-install-and-generate-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <executable>${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh</executable>
+ <arguments>
+ <argument>${thrift.install.env}</argument>
+ </arguments>
+ <skip>false</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
  <profile>
  <id>m1-mac</id>
  <properties>
@@ -108,27 +136,6 @@
 
  <build>
  <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.6.0</version>
- <executions>
- <execution>
- <id>thrift-install-and-generate-source</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <executable>${project.parent.basedir}/src/main/thrift/bin/thrift_binary.sh</executable>
- <arguments>
- <argument>${thrift.install.env}</argument>
- </arguments>
- <skip>false</skip>
- </configuration>
- </plugin>
  <plugin>
  <groupId>org.jacoco</groupId>
  <artifactId>jacoco-maven-plugin</artifactId>

pom.xml

@@ -232,6 +232,7 @@
  <springboot.version>2.7.3</springboot.version>
  <spring.shell.version>2.1.1</spring.shell.version>
  <snappy.version>1.1.8.3</snappy.version>
+ <thrift.executable>/usr/local/bin/thrift</thrift.executable>
  </properties>
 
  <scm>