From e32667ccb3201c82346725692a8a44ed8784832b Mon Sep 17 00:00:00 2001 From: xBis7 Date: Mon, 22 Jan 2024 12:02:45 +0200 Subject: [PATCH 1/8] Move hadoop docker scripts under the main source code --- hadoop-dist/pom.xml | 94 +++++++++++++++++++ hadoop-dist/src/main/compose/README.md | 31 ++++++ hadoop-dist/src/main/compose/hadoop/.env | 18 ++++ hadoop-dist/src/main/compose/hadoop/config | 50 ++++++++++ .../main/compose/hadoop/docker-compose.yaml | 46 +++++++++ hadoop-dist/src/main/docker/Dockerfile | 20 ++++ pom.xml | 14 +++ 7 files changed, 273 insertions(+) create mode 100644 hadoop-dist/src/main/compose/README.md create mode 100644 hadoop-dist/src/main/compose/hadoop/.env create mode 100644 hadoop-dist/src/main/compose/hadoop/config create mode 100644 hadoop-dist/src/main/compose/hadoop/docker-compose.yaml create mode 100644 hadoop-dist/src/main/docker/Dockerfile diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index dc92d44010961..3323bcafbf3f5 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -29,6 +29,13 @@ Apache Hadoop Distribution jar + + UTF-8 + true + docker-hadoop-3 + true + + @@ -151,6 +158,43 @@ + + maven-resources-plugin + + + copy-compose-files + package + + copy-resources + + + ${project.build.directory}/hadoop-${project.version}/compose + + + src/main/compose + true + + + + + + copy-and-filter-dockerfile + package + + copy-resources + + + ${project.build.directory}/hadoop-${project.version} + + + src/main/docker + true + + + + + + @@ -230,6 +274,56 @@ + + docker-build + + + + io.fabric8 + docker-maven-plugin + + + + build + + package + + + + + + ${docker.image} + + + ${project.build.directory}/hadoop-${project.version} + + + + + + + + + + + docker-push + + + + io.fabric8 + docker-maven-plugin + + + + push + + package + + + + + + diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md new file mode 100644 index 0000000000000..65a48b20767b6 --- /dev/null +++ b/hadoop-dist/src/main/compose/README.md @@ -0,0 +1,31 @@ +## Hadoop Docker + +This directory contains Docker Compose definitions for running the current version of Hadoop in a multi-node docker environment. + +This is meant for testing code changes locally and debugging. + +The image used by the Docker setup is built as part of the maven lifecycle. + +In order to start the docker environment you need to do the following +* Build the project, which will also build the docker image + * ```shell + > mvn clean install -Dmaven.javadoc.skip=true -DskipTests -DskipShade -Pdist,src + ``` +* From the project root, navigate under the docker-compose dir under the generated dist directory + * ```shell + > cd hadoop-dist/target/hadoop-/compose/hadoop + ``` +* Start the docker environment + * ```shell + > docker-compose up -d --scale datanode=3 + ``` +* Connect to a container to execute commands + * ```shell + > docker exec -it hadoop_datanode_1 bash + bash-4.2$ hdfs dfs -mkdir /test + ``` + +### Config files + +To add or remove properties from the `core-site.xml`, `hdfs-site.xml`, etc. files used in the docker environment, +simply edit the `config` file before starting the containers. The changes will be persisted in the docker environment. diff --git a/hadoop-dist/src/main/compose/hadoop/.env b/hadoop-dist/src/main/compose/hadoop/.env new file mode 100644 index 0000000000000..838efcdebf792 --- /dev/null +++ b/hadoop-dist/src/main/compose/hadoop/.env @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HADOOP_IMAGE=apache/hadoop +HADOOP_RUNNER_VERSION=${docker.hadoop-runner.version} +HADOOP_RUNNER_IMAGE=apache/hadoop-runner diff --git a/hadoop-dist/src/main/compose/hadoop/config b/hadoop-dist/src/main/compose/hadoop/config new file mode 100644 index 0000000000000..1fac879f7eaea --- /dev/null +++ b/hadoop-dist/src/main/compose/hadoop/config @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode + +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 + +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME + +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle + +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false + +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/hadoop-dist/src/main/compose/hadoop/docker-compose.yaml b/hadoop-dist/src/main/compose/hadoop/docker-compose.yaml new file mode 100644 index 0000000000000..f999c39300e7d --- /dev/null +++ b/hadoop-dist/src/main/compose/hadoop/docker-compose.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +x-common-config: + &common-config + image: ${HADOOP_RUNNER_IMAGE}:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + env_file: + - ./config + +services: + namenode: + <<: *common-config + hostname: namenode + command: ["hdfs", "namenode"] + ports: + - 9870:9870 + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + datanode: + <<: *common-config + command: ["hdfs", "datanode"] + resourcemanager: + <<: *common-config + hostname: resourcemanager + command: ["yarn", "resourcemanager"] + ports: + - 8088:8088 + nodemanager: + <<: *common-config + command: ["yarn", "nodemanager"] diff --git a/hadoop-dist/src/main/docker/Dockerfile b/hadoop-dist/src/main/docker/Dockerfile new file mode 100644 index 0000000000000..df858e240ee98 --- /dev/null +++ b/hadoop-dist/src/main/docker/Dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/hadoop-runner:@docker.hadoop-runner.version@ + +ADD --chown=hadoop . /opt/hadoop + +WORKDIR /opt/hadoop diff --git a/pom.xml b/pom.xml index 964c23c6cb56d..a8320f5507079 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x 3.5.0-SNAPSHOT + apache/hadoop:${project.version} + apache.snapshots.https Apache Development Snapshot Repository https://repository.apache.org/content/repositories/snapshots @@ -119,6 +121,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x 1.1.1 3.10.1 2.7.10 + 0.29.0 bash @@ -150,6 +153,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x maven-dependency-plugin ${maven-dependency-plugin.version} + + io.fabric8 + docker-maven-plugin + ${docker-maven-plugin.version} + org.apache.maven.plugins maven-enforcer-plugin @@ -892,5 +900,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/x + + docker-build + + ${user.name}/hadoop:${project.version} + + From a18a2fb2fd614c1b3025bf9e31723c25f97ec884 Mon Sep 17 00:00:00 2001 From: xBis7 Date: Mon, 22 Jan 2024 12:06:21 +0200 Subject: [PATCH 2/8] remove space --- hadoop-dist/src/main/compose/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md index 65a48b20767b6..ac053209220d4 100644 --- a/hadoop-dist/src/main/compose/README.md +++ b/hadoop-dist/src/main/compose/README.md @@ -6,7 +6,7 @@ This is meant for testing code changes locally and debugging. The image used by the Docker setup is built as part of the maven lifecycle. -In order to start the docker environment you need to do the following +In order to start the docker environment you need to do the following * Build the project, which will also build the docker image * ```shell > mvn clean install -Dmaven.javadoc.skip=true -DskipTests -DskipShade -Pdist,src From 5810004edb3b85e9c3cfd63f935f3328ab59c37a Mon Sep 17 00:00:00 2001 From: xBis7 Date: Mon, 22 Jan 2024 12:28:20 +0200 Subject: [PATCH 3/8] add README license --- hadoop-dist/src/main/compose/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md index ac053209220d4..915772d7ce18a 100644 --- a/hadoop-dist/src/main/compose/README.md +++ b/hadoop-dist/src/main/compose/README.md @@ -1,3 +1,20 @@ + + ## Hadoop Docker This directory contains Docker Compose definitions for running the current version of Hadoop in a multi-node docker environment. From 464c37847a3fe848a8e16fda3caf3261e3d97212 Mon Sep 17 00:00:00 2001 From: xBis7 Date: Tue, 23 Jan 2024 13:27:34 +0200 Subject: [PATCH 4/8] fix CI Dockerfile error --- hadoop-dist/src/main/docker/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-dist/src/main/docker/Dockerfile b/hadoop-dist/src/main/docker/Dockerfile index df858e240ee98..2f4d9aa5ecb19 100644 --- a/hadoop-dist/src/main/docker/Dockerfile +++ b/hadoop-dist/src/main/docker/Dockerfile @@ -15,6 +15,8 @@ FROM apache/hadoop-runner:@docker.hadoop-runner.version@ -ADD --chown=hadoop . /opt/hadoop +COPY . /opt/hadoop WORKDIR /opt/hadoop + +RUN sudo chown -R hadoop:users /opt/hadoop From 03d600f0a51e9cefa7282856215c77e63196583b Mon Sep 17 00:00:00 2001 From: xBis7 Date: Fri, 2 Feb 2024 16:07:41 +0200 Subject: [PATCH 5/8] HadoopDocker.md --- .../src/site/markdown/HadoopDocker.md | 63 +++++++++++++++++++ .../src/site/markdown/SingleCluster.md.vm | 6 ++ hadoop-dist/src/main/compose/README.md | 48 -------------- 3 files changed, 69 insertions(+), 48 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md delete mode 100644 hadoop-dist/src/main/compose/README.md diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md b/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md new file mode 100644 index 0000000000000..33ba7d4b0bae1 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md @@ -0,0 +1,63 @@ + + +## Hadoop Docker + +### Official releases + +There are special branches for running specific hadoop version in docker. +These branches, use the files from official hadoop releases for running hadoop inside the containers. + +E.g. +* [docker-hadoop-runner-latest](https://github.com/apache/hadoop/tree/docker-hadoop-runner-latest) +* [docker-hadoop-runner-jdk11](https://github.com/apache/hadoop/tree/docker-hadoop-runner-jdk11) +* [docker-hadoop-runner-jdk8](https://github.com/apache/hadoop/tree/docker-hadoop-runner-jdk8) +* [docker-hadoop-runner](https://github.com/apache/hadoop/tree/docker-hadoop-runner) +* [docker-hadoop-3](https://github.com/apache/hadoop/tree/docker-hadoop-3) +* [docker-hadoop-2](https://github.com/apache/hadoop/tree/docker-hadoop-2) + +### Source code + +There is a setup under `hadoop-dist` that contains Docker Compose definitions +for running the current version of Hadoop in a multi-node docker environment. + +This is meant for testing code changes locally and debugging. + +The base image used by the Docker setup is built as part of the maven lifecycle. +The distribution files generated while building the project with the `-Pdist` profile enabled, +will be used for running hadoop inside the containers. + +In order to start the docker environment you need to do the following +* Build the project, using the `-Pdist` profile + ```shell + > mvn clean install -Dmaven.javadoc.skip=true -DskipTests -DskipShade -Pdist,src + ``` +* From the project root, navigate under the docker-compose dir under the generated dist directory + ```shell + > cd hadoop-dist/target/hadoop-/compose/hadoop + ``` +* Start the docker environment + ```shell + > docker-compose up -d --scale datanode=3 + ``` +* Connect to a container to execute commands + ```shell + > docker exec -it hadoop_datanode_1 bash + bash-4.2$ hdfs dfs -mkdir /test + ``` + +### Config files + +To add or remove properties from the `core-site.xml`, `hdfs-site.xml`, etc. files used in the docker environment, +simply edit the `config` file before starting the containers. The changes will be persisted in the docker environment. diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm index 8153dce5c3f82..ad0698a03eb07 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm +++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm @@ -236,3 +236,9 @@ Fully-Distributed Operation --------------------------- For information on setting up fully-distributed, non-trivial clusters see [Cluster Setup](./ClusterSetup.html). + +Hadoop in Docker containers +--------------------------- + +For information on setting up hadoop in docker, using either official releases or the main source code, +check [Hadoop Docker](./HadoopDocker.html). diff --git a/hadoop-dist/src/main/compose/README.md b/hadoop-dist/src/main/compose/README.md deleted file mode 100644 index 915772d7ce18a..0000000000000 --- a/hadoop-dist/src/main/compose/README.md +++ /dev/null @@ -1,48 +0,0 @@ - - -## Hadoop Docker - -This directory contains Docker Compose definitions for running the current version of Hadoop in a multi-node docker environment. - -This is meant for testing code changes locally and debugging. - -The image used by the Docker setup is built as part of the maven lifecycle. - -In order to start the docker environment you need to do the following -* Build the project, which will also build the docker image - * ```shell - > mvn clean install -Dmaven.javadoc.skip=true -DskipTests -DskipShade -Pdist,src - ``` -* From the project root, navigate under the docker-compose dir under the generated dist directory - * ```shell - > cd hadoop-dist/target/hadoop-/compose/hadoop - ``` -* Start the docker environment - * ```shell - > docker-compose up -d --scale datanode=3 - ``` -* Connect to a container to execute commands - * ```shell - > docker exec -it hadoop_datanode_1 bash - bash-4.2$ hdfs dfs -mkdir /test - ``` - -### Config files - -To add or remove properties from the `core-site.xml`, `hdfs-site.xml`, etc. files used in the docker environment, -simply edit the `config` file before starting the containers. The changes will be persisted in the docker environment. From e9a25c68fd64b87f7bb3c49252ea1a75f4b522ee Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Thu, 10 Oct 2024 19:29:53 +0300 Subject: [PATCH 6/8] fix Dockerfile ci error --- hadoop-dist/src/main/docker/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-dist/src/main/docker/Dockerfile b/hadoop-dist/src/main/docker/Dockerfile index 2f4d9aa5ecb19..7ea60ed7f658f 100644 --- a/hadoop-dist/src/main/docker/Dockerfile +++ b/hadoop-dist/src/main/docker/Dockerfile @@ -19,4 +19,8 @@ COPY . /opt/hadoop WORKDIR /opt/hadoop -RUN sudo chown -R hadoop:users /opt/hadoop +USER root + +RUN chown -R hadoop:users /opt/hadoop + +USER hadoop From 1343de866d156e06e8d0e962be6ca4e363d45fc5 Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Fri, 18 Oct 2024 14:43:02 +0300 Subject: [PATCH 7/8] update README + change hadoop-runner version --- .../hadoop-common/src/site/markdown/HadoopDocker.md | 8 ++++---- hadoop-dist/pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md b/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md index 33ba7d4b0bae1..fb3da82a6bdd7 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md @@ -14,10 +14,10 @@ ## Hadoop Docker -### Official releases +### hadoop-runner-* images -There are special branches for running specific hadoop version in docker. -These branches, use the files from official hadoop releases for running hadoop inside the containers. +There are special branches for running hadoop in docker. +These branches contain scripts that set up base images that can be used for running any Hadoop version. E.g. * [docker-hadoop-runner-latest](https://github.com/apache/hadoop/tree/docker-hadoop-runner-latest) @@ -27,7 +27,7 @@ E.g. * [docker-hadoop-3](https://github.com/apache/hadoop/tree/docker-hadoop-3) * [docker-hadoop-2](https://github.com/apache/hadoop/tree/docker-hadoop-2) -### Source code +### Running from the source code There is a setup under `hadoop-dist` that contains Docker Compose definitions for running the current version of Hadoop in a multi-node docker environment. diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 3323bcafbf3f5..2229a89fe7a0e 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -32,7 +32,7 @@ UTF-8 true - docker-hadoop-3 + jdk8 true From 08987d1cdff8a6fd49e324498b59d25e0d4dac6d Mon Sep 17 00:00:00 2001 From: Christos Bisias Date: Fri, 18 Oct 2024 15:19:00 +0300 Subject: [PATCH 8/8] update HadoopDocker.md --- .../hadoop-common/src/site/markdown/HadoopDocker.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md b/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md index fb3da82a6bdd7..c92a4c64de1c9 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/HadoopDocker.md @@ -14,18 +14,23 @@ ## Hadoop Docker -### hadoop-runner-* images +### Running from existing setups There are special branches for running hadoop in docker. -These branches contain scripts that set up base images that can be used for running any Hadoop version. -E.g. +The `docker-hadoop-runner*` branches contain scripts that set up base images that can be used for running any Hadoop version. + * [docker-hadoop-runner-latest](https://github.com/apache/hadoop/tree/docker-hadoop-runner-latest) * [docker-hadoop-runner-jdk11](https://github.com/apache/hadoop/tree/docker-hadoop-runner-jdk11) * [docker-hadoop-runner-jdk8](https://github.com/apache/hadoop/tree/docker-hadoop-runner-jdk8) * [docker-hadoop-runner](https://github.com/apache/hadoop/tree/docker-hadoop-runner) + +The `docker-hadoop*` branches can be used for running a specific version. + * [docker-hadoop-3](https://github.com/apache/hadoop/tree/docker-hadoop-3) + * `hadoop-3.3.6` * [docker-hadoop-2](https://github.com/apache/hadoop/tree/docker-hadoop-2) + * `hadoop-2.10.2` ### Running from the source code