build.gradle

/*
 * SPDX-License-Identifier: Apache-2.0
 *
 * The OpenSearch Contributors require contributions made to
 * this file be licensed under the Apache-2.0 license or a
 * compatible open source license.
 *
 * Modifications Copyright OpenSearch Contributors. See
 * GitHub history for details.
 */

/*
 * Licensed to Elasticsearch under one or more contributor
 * license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright
 * ownership. Elasticsearch licenses this file to you under
 * the Apache License, Version 2.0 (the "License"); you may
 * not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

import java.nio.charset.StandardCharsets;
import java.io.ByteArrayOutputStream;

import com.avast.gradle.dockercompose.tasks.ComposePull
import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
import org.apache.tools.ant.taskdefs.condition.Os
import org.opensearch.gradle.BuildPlugin
import org.opensearch.gradle.Version
import org.opensearch.gradle.VersionProperties
import org.opensearch.gradle.info.BuildParams
import org.opensearch.gradle.plugin.PluginBuildPlugin
import org.opensearch.gradle.tar.SymbolicLinkPreservingTar
import org.gradle.plugins.ide.eclipse.model.AccessRule
import org.gradle.plugins.ide.eclipse.model.EclipseJdt
import org.gradle.plugins.ide.eclipse.model.SourceFolder
import org.gradle.api.Project;
import org.gradle.api.internal.tasks.testing.junit.JUnitTestFramework
import org.gradle.process.ExecResult;

import static org.opensearch.gradle.util.GradleUtils.maybeConfigure

plugins {
  id 'lifecycle-base'
  id 'opensearch.docker-support'
  id 'opensearch.global-build-info'
  id "com.diffplug.spotless" version "6.15.0" apply false
  id "org.gradle.test-retry" version "1.5.1" apply false
  id "test-report-aggregation"
  id 'jacoco-report-aggregation'
}

apply from: 'gradle/build-complete.gradle'
apply from: 'gradle/runtime-jdk-provision.gradle'
apply from: 'gradle/ide.gradle'
apply from: 'gradle/forbidden-dependencies.gradle'
apply from: 'gradle/formatting.gradle'
apply from: 'gradle/local-distribution.gradle'
apply from: 'gradle/fips.gradle'
apply from: 'gradle/run.gradle'
apply from: 'gradle/missing-javadoc.gradle'
apply from: 'gradle/code-coverage.gradle'

// common maven publishing configuration
allprojects {
  group = 'org.opensearch'
  version = VersionProperties.getOpenSearch()
  description = "OpenSearch subproject ${project.path}"

  afterEvaluate {
    project.tasks.withType(Test) { task ->
      // This is so hacky: now, by default, test tasks uses JUnit framework and always includes 'junit'
      // JARs from the Gradle distribution (no ways to override this behavior). It causes JAR hell on test 
      // classpath, example of the report:
      //
      //  jar1: /home/ubuntu/.gradle/caches/modules-2/files-2.1/junit/junit/4.13.2/8ac9e16d933b6fb43bc7f576336b8f4d7eb5ba12/junit-4.13.2.jar
      //  jar2: /home/ubuntu/.gradle/wrapper/dists/gradle-8.0-rc-1-all/2p8rgxxewg8l61n1p3vrzr9s8/gradle-8.0-rc-1/lib/junit-4.13.2.jar
      // 
      task.getTestFrameworkProperty().convention(getProviderFactory().provider(() -> new JUnitTestFramework(task, task.getFilter(), false)));
    }
  }
}

configure(allprojects - project(':distribution:archives:integ-test-zip')) {
  project.pluginManager.withPlugin('nebula.maven-base-publish') {
    if (project.pluginManager.hasPlugin('opensearch.build') == false) {
      throw new GradleException("Project ${path} publishes a pom but doesn't apply the build plugin.")
    }
  }
}

subprojects {
  // Default to the apache license
  project.ext.licenseName = 'The Apache Software License, Version 2.0'
  project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'

  // we only use maven publish to add tasks for pom generation
  plugins.withType(MavenPublishPlugin).whenPluginAdded {
    publishing {
      publications {
        // add license information to generated poms
        all {
          pom.withXml { XmlProvider xml ->
            Node node = xml.asNode()
            node.appendNode('inceptionYear', '2021')

            Node license = node.appendNode('licenses').appendNode('license')
            license.appendNode('name', project.licenseName)
            license.appendNode('url', project.licenseUrl)
            license.appendNode('distribution', 'repo')

            Node developer = node.appendNode('developers').appendNode('developer')
            developer.appendNode('name', 'OpenSearch')
            developer.appendNode('url', 'https:/opensearch-project/OpenSearch')
          }
        }
      }
      repositories {
        maven {
          name = 'test'
          url = "${rootProject.buildDir}/local-test-repo"
        }
        maven {
          name = 'Snapshots'
          url = 'https://aws.oss.sonatype.org/content/repositories/snapshots'
          credentials {
            username "$System.env.SONATYPE_USERNAME"
            password "$System.env.SONATYPE_PASSWORD"
          }
        }
      }
    }
  }

  plugins.withType(BuildPlugin).whenPluginAdded {
    project.licenseFile = project.rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
    project.noticeFile = project.rootProject.file('NOTICE.txt')
  }
}

tasks.register("updateCIBwcVersions") {
  doLast {
    File yml = file(".ci/bwcVersions")
    yml.text = ""
    yml << "BWC_VERSION:\n"
    BuildParams.bwcVersions.indexCompatible.each {
      yml << "  - \"$it\"\n"
    }
  }
}

// build metadata from previous build, contains eg hashes for bwc builds
String buildMetadataValue = System.getenv('BUILD_METADATA')
if (buildMetadataValue == null) {
  buildMetadataValue = ''
}
Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectEntries {
  def (String key, String value) = it.split('=')
  return [key, value]
}

    /**
     * Using 'git' command line (if available), tries to fetch the commit date of the current revision
     * @return commit date of the current revision or 0 if it is not available
     */
long gitRevisionDate = {
  // Try to get last commit date as Unix timestamp
  try (ByteArrayOutputStream stdout = new ByteArrayOutputStream()) {
     ExecResult result = project.exec(spec -> {
        spec.setIgnoreExitValue(true);
        spec.setStandardOutput(stdout);
        spec.commandLine("git", "log", "-1", "--format=%ct");
     });

    if (result.getExitValue() == 0) {
      return Long.parseLong(stdout.toString(StandardCharsets.UTF_8).replaceAll("\\s", "")) * 1000; /* seconds to millis */
    }
  } catch (IOException | GradleException | NumberFormatException ex) {
    /* fall back to default Unix epoch timestamp */
  }

  return 0;
}()


// injecting groovy property variables into all projects
allprojects {
  project.ext {
    // for ide hacks...
    isEclipse = System.getProperty("eclipse.launcher") != null ||   // Detects gradle launched from Eclipse's IDE
      System.getProperty("eclipse.application") != null ||    // Detects gradle launched from the Eclipse compiler server
      gradle.startParameter.taskNames.contains('eclipse') ||  // Detects gradle launched from the command line to do eclipse stuff
      gradle.startParameter.taskNames.contains('cleanEclipse')

    buildMetadata = buildMetadataMap
  }
}

tasks.register("verifyVersions") {
  doLast {
    if (gradle.startParameter.isOffline()) {
      throw new GradleException("Must run in online mode to verify versions")
    }
    // Read the list from maven central.
    // Fetch the metadata and parse the xml into Version instances because it's more straight forward here
    // rather than bwcVersion ( VersionCollection ).
    new URL('https://repo1.maven.org/maven2/org/opensearch/opensearch/maven-metadata.xml').openStream().withStream { s ->
      BuildParams.bwcVersions.compareToAuthoritative(
        new XmlParser().parse(s)
          .versioning.versions.version
          .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
          .collect { Version.fromString(it) }
      )
    }
    String ciYml = file(".ci/bwcVersions").text
    BuildParams.bwcVersions.indexCompatible.each {
      if (ciYml.contains("\"$it\"\n") == false) {
        throw new Exception(".ci/bwcVersions is outdated, run `./gradlew updateCIBwcVersions` and check in the results");
      }
    }
  }
}

/*
 * When adding backcompat behavior that spans major versions, temporarily
 * disabling the backcompat tests is necessary. This flag controls
 * the enabled state of every bwc task. It should be set back to true
 * after the backport of the backcompat code is complete.
 */

boolean bwc_tests_enabled = true
/* place an issue link here when committing bwc changes */
final String bwc_tests_disabled_issue = ""
if (bwc_tests_enabled == false) {
  if (bwc_tests_disabled_issue.isEmpty()) {
    throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
  }
  println "========================= WARNING ========================="
  println "         Backwards compatibility tests are disabled!"
  println "See ${bwc_tests_disabled_issue}"
  println "==========================================================="
}
if (project.gradle.startParameter.taskNames.find { it.startsWith("checkPart") } != null) {
  // Disable BWC tests for checkPart* tasks as it's expected that this will run un it's own check
  bwc_tests_enabled = false
}

subprojects {
  ext.bwc_tests_enabled = bwc_tests_enabled
}

tasks.register("verifyBwcTestsEnabled") {
  doLast {
    if (bwc_tests_enabled == false) {
      throw new GradleException('Bwc tests are disabled. They must be re-enabled after completing backcompat behavior backporting.')
    }
  }
}

tasks.register("branchConsistency") {
  description 'Ensures this branch is internally consistent. For example, that versions constants match released versions.'
  group 'Verification'
  dependsOn ":verifyVersions", ":verifyBwcTestsEnabled"
}

allprojects {
  // configure compiler options
  tasks.withType(JavaCompile).configureEach { JavaCompile compile ->
    options.fork = true

    configure(options.forkOptions) {
      memoryMaximumSize = project.property('options.forkOptions.memoryMaximumSize')
    }

    // See please https://bugs.openjdk.java.net/browse/JDK-8209058
    if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_11) {
      compile.options.compilerArgs << '-Werror'
    }
    compile.options.compilerArgs << '-Xlint:auxiliaryclass'
    compile.options.compilerArgs << '-Xlint:cast'
    compile.options.compilerArgs << '-Xlint:classfile'
    compile.options.compilerArgs << '-Xlint:dep-ann'
    compile.options.compilerArgs << '-Xlint:divzero'
    compile.options.compilerArgs << '-Xlint:empty'
    compile.options.compilerArgs << '-Xlint:exports'
    compile.options.compilerArgs << '-Xlint:fallthrough'
    compile.options.compilerArgs << '-Xlint:finally'
    compile.options.compilerArgs << '-Xlint:module'
    compile.options.compilerArgs << '-Xlint:opens'
    compile.options.compilerArgs << '-Xlint:overloads'
    compile.options.compilerArgs << '-Xlint:overrides'
    compile.options.compilerArgs << '-Xlint:-processing'
    compile.options.compilerArgs << '-Xlint:rawtypes'
    compile.options.compilerArgs << '-Xlint:removal'
    compile.options.compilerArgs << '-Xlint:requires-automatic'
    compile.options.compilerArgs << '-Xlint:requires-transitive-automatic'
    compile.options.compilerArgs << '-Xlint:static'
    compile.options.compilerArgs << '-Xlint:unchecked'
    compile.options.compilerArgs << '-Xlint:varargs'
    compile.options.compilerArgs << '-Xlint:preview'
    // TODO: disabled warnings: path, serial, options, deprecation, try
    // -path because gradle will send in paths that don't always exist.
    // -missing because we have tons of missing @returns and @param.
    // -serial because we don't use java serialization.
    compile.options.compilerArgs << '-Xdoclint:accessibility'
    compile.options.compilerArgs << '-Xdoclint:html'
    compile.options.compilerArgs << '-Xdoclint:reference'
    compile.options.compilerArgs << '-Xdoclint:syntax'
  }

  // ignore missing javadocs
  tasks.withType(Javadoc).configureEach { Javadoc javadoc ->
    // the -quiet here is because of a bug in gradle, in that adding a string option
    // by itself is not added to the options. By adding quiet, both this option and
    // the "value" -quiet is added, separated by a space. This is ok since the javadoc
    // command already adds -quiet, so we are just duplicating it
    // see https://discuss.gradle.org/t/add-custom-javadoc-option-that-does-not-take-an-argument/5959
    javadoc.options.encoding = 'UTF8'
    javadoc.options.addStringOption('Xdoclint:all,-missing', '-quiet')
    boolean failOnJavadocWarning = project.ext.has('failOnJavadocWarning') ? project.ext.get('failOnJavadocWarning') : true
    if (failOnJavadocWarning) {
      javadoc.options.addStringOption('Xwerror', '-quiet')
    }
    javadoc.options.tags = ["opensearch.internal", "opensearch.api", "opensearch.experimental"]
    javadoc.options.addStringOption("-release", targetCompatibility.majorVersion)
  }

  // support for reproducible builds
  tasks.withType(AbstractArchiveTask).configureEach { task ->
    // ignore file timestamps
    // be consistent in archive file order
    task.preserveFileTimestamps = false
    task.reproducibleFileOrder = true
    if (task instanceof SymbolicLinkPreservingTar) {
      // Replace file timestamps with latest Git revision date (if available)
      task.lastModifiedTimestamp = gitRevisionDate
    }
  }

  project.afterEvaluate {
    // Handle javadoc dependencies across projects. Order matters: the linksOffline for
    // org.opensearch:opensearch must be the last one or all the links for the
    // other packages (e.g org.opensearch.client) will point to server rather than
    // their own artifacts.
    if (project.plugins.hasPlugin(BuildPlugin) || project.plugins.hasPlugin(PluginBuildPlugin)) {
      String artifactsHost = VersionProperties.getOpenSearch().endsWith("-SNAPSHOT")
        ? "https://artifacts.opensearch.org/snapshots/"
        : "https://artifacts.opensearch.org/releases/"
      Closure sortClosure = { a, b -> b.group <=> a.group }
      Closure depJavadocClosure = { shadowed, dep ->
        if ((dep instanceof ProjectDependency) == false) {
          return
        }
        Project upstreamProject = dep.dependencyProject
        if (upstreamProject == null) {
          return
        }
        if (shadowed) {
          /*
           * Include the source of shadowed upstream projects so we don't
           * have to publish their javadoc.
           */
          project.evaluationDependsOn(upstreamProject.path)
          project.javadoc.source += upstreamProject.javadoc.source
          /*
           * Instead we need the upstream project's javadoc classpath so
           * we don't barf on the classes that it references.
           */
          project.javadoc.classpath += upstreamProject.javadoc.classpath
        } else {
          // Link to non-shadowed dependant projects
          project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
          String externalLinkName = upstreamProject.archivesBaseName
          String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version
          String projectRelativePath = project.relativePath(upstreamProject.buildDir)
          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/"
        }
      }
      boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)
      project.configurations.implementation.dependencies
        .findAll()
        .toSorted(sortClosure)
        .each({ c -> depJavadocClosure(hasShadow, c) })
      project.configurations.compileOnly.dependencies
        .findAll()
        .toSorted(sortClosure)
        .each({ c -> depJavadocClosure(false, c) })
      if (hasShadow) {
        // include any dependencies for shadow JAR projects that are *not* bundled in the shadow JAR
        project.configurations.shadow.dependencies
          .findAll()
          .toSorted(sortClosure)
          .each({ c -> depJavadocClosure(false, c) })
      }
    }
  }
}

// Ensure similar tasks in dependent projects run first. The projectsEvaluated here is
// important because, while dependencies.all will pickup future dependencies,
// it is not necessarily true that the task exists in both projects at the time
// the dependency is added.
gradle.projectsEvaluated {
  allprojects {
    project.tasks.withType(JavaForkOptions) {
      maxHeapSize project.property('options.forkOptions.memoryMaximumSize')
    }

    if (project.path == ':test:framework') {
      // :test:framework:test cannot run before and after :server:test
      return
    }
    if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) {
      integTest.mustRunAfter test
    }

    project.tasks.withType(Test) { task ->
      if (task != null) {
        if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
          task.jvmArgs += ["-Djava.security.manager=allow"]
        }
      }
    }

    configurations.matching { it.canBeResolved }.all { Configuration configuration ->
      dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
        Project upstreamProject = dep.dependencyProject
        if (upstreamProject != null) {
          if (project.path == upstreamProject.path) {
            // TODO: distribution integ tests depend on themselves (!), fix that
            return
          }
          for (String taskName : ['test', 'integTest']) {
            Task task = project.tasks.findByName(taskName)
            Task upstreamTask = upstreamProject.tasks.findByName(taskName)
            if (task != null && upstreamTask != null) {
              task.shouldRunAfter(upstreamTask)
            }
          }
        }
      }
    }
  }

  dependencies {
    subprojects.findAll { it.pluginManager.hasPlugin('java') }.forEach {
      testReportAggregation it
    }
    subprojects.findAll { it.pluginManager.hasPlugin('jacoco') }.forEach {
     jacocoAggregation it
    }
  }
}

// test retry configuration
subprojects {
  apply plugin: "org.gradle.test-retry"
  tasks.withType(Test).configureEach {
    retry {
      if (BuildParams.isCi()) {
        maxRetries = 3
        maxFailures = 10
      }
      failOnPassedAfterRetry = false
    }
  }
}

// eclipse configuration
allprojects {
  apply plugin: 'eclipse'

  // Name all the non-root projects after their path so that paths get grouped together when imported into eclipse.
  if (path != ':') {
    eclipse.project.name = path
    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
      eclipse.project.name = eclipse.project.name.replace(':', '_')
    }
  }

  plugins.withType(JavaBasePlugin) {
    eclipse.classpath.defaultOutputDir = file('build-eclipse')
    eclipse.classpath.file.whenMerged { classpath ->
      // give each source folder a unique corresponding output folder
      int i = 0;
      classpath.entries.findAll { it instanceof SourceFolder }.each { folder ->
        i++;
        folder.output = "build-eclipse/" + i
      }
    }
  }
  /*
   * Allow accessing com/sun/net/httpserver in projects that have
   * configured forbidden apis to allow it.
   */
  plugins.withType(ForbiddenApisPlugin) {
    eclipse.classpath.file.whenMerged { classpath ->
      if (false == forbiddenApisTest.bundledSignatures.contains('jdk-non-portable')) {
        classpath.entries
          .findAll { it.kind == "con" && it.toString().contains("org.eclipse.jdt.launching.JRE_CONTAINER") }
          .each {
            it.accessRules.add(new AccessRule("accessible", "com/sun/net/httpserver/*"))
          }
      }
    }
  }

  File licenseHeaderFile
  licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/license-header.txt')

  String lineSeparator = Os.isFamily(Os.FAMILY_WINDOWS) ? '\\\\r\\\\n' : '\\\\n'
  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace(System.lineSeparator(), lineSeparator)
  tasks.register('copyEclipseSettings', Copy) {
    mustRunAfter 'wipeEclipseSettings'
    // TODO: "package this up" for external builds
    from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings')
    into '.settings'
    filter { it.replaceAll('@@LICENSE_HEADER_TEXT@@', licenseHeader) }
  }
  // otherwise .settings is not nuked entirely
  tasks.register('wipeEclipseSettings', Delete) {
    delete '.settings'
  }
  tasks.named('cleanEclipse') { dependsOn 'wipeEclipseSettings' }
  // otherwise the eclipse merging is *super confusing*
  tasks.named('eclipse') { dependsOn 'cleanEclipse', 'copyEclipseSettings' }

  afterEvaluate {
    tasks.findByName("eclipseJdt")?.configure {
      dependsOn 'copyEclipseSettings'
    }
  }
}

wrapper {
  distributionType = 'ALL'
  doLast {
    def sha256Sum = new String(new URL(getDistributionUrl() + ".sha256").bytes)
    propertiesFile << "distributionSha256Sum=${sha256Sum}\n"
    println "Added checksum to wrapper properties"
    // Update build-tools to reflect the Gradle upgrade
    // TODO: we can remove this once we have tests to make sure older versions work.
    project(':build-tools').file('src/main/resources/minimumGradleVersion').text = gradleVersion + "\n"
    println "Updated minimum Gradle Version"
  }
}

gradle.projectsEvaluated {
  subprojects {
    /*
     * Remove assemble/dependenciesInfo on all qa projects because we don't
     * need to publish artifacts for them.
     */
    if (project.name.equals('qa') || project.path.contains(':qa:')) {
      maybeConfigure(project.tasks, 'assemble') {
        it.enabled = false
      }
      maybeConfigure(project.tasks, 'dependenciesInfo') {
        it.enabled = false
      }
    }
  }
  // Having the same group and name for distinct projects causes Gradle to consider them equal when resolving
  // dependencies leading to hard to debug failures. Run a check across all project to prevent this from happening.
  // see: https:/gradle/gradle/issues/847
  Map coordsToProject = [:]
  project.allprojects.forEach { p ->
    String coords = "${p.group}:${p.name}"
    if (false == coordsToProject.putIfAbsent(coords, p)) {
      throw new GradleException(
        "Detected that two projects: ${p.path} and ${coordsToProject[coords].path} " +
          "have the same name and group: ${coords}. " +
          "This doesn't currently work correctly in Gradle, see: " +
          "https:/gradle/gradle/issues/847"
      )
    }
  }
}

allprojects {
  tasks.register('resolveAllDependencies', org.opensearch.gradle.ResolveAllDependencies) {
    configs = project.configurations
    if (project.path.contains("fixture")) {
      dependsOn tasks.withType(ComposePull)
    }
  }

  // helper task to print direct dependencies of a single task
  project.tasks.addRule("Pattern: <taskName>Dependencies") { String taskName ->
    if (taskName.endsWith("Dependencies") == false) {
      return
    }
    if (project.tasks.findByName(taskName) != null) {
      return
    }
    String realTaskName = taskName.substring(0, taskName.length() - "Dependencies".length())
    Task realTask = project.tasks.findByName(realTaskName)
    if (realTask == null) {
      return
    }
    project.tasks.register(taskName) {
      doLast {
        println("${realTask.path} dependencies:")
        for (Task dep : realTask.getTaskDependencies().getDependencies(realTask)) {
          println("  - ${dep.path}")
        }
      }
    }
  }

  def checkPart1 = tasks.register('checkPart1')
  def checkPart2 = tasks.register('checkPart2')
  plugins.withId('lifecycle-base') {
    checkPart1.configure { dependsOn 'check' }
  }
}

subprojects {
  project.ext.disableTasks = { String... tasknames ->
    for (String taskname : tasknames) {
      project.tasks.named(taskname).configure { onlyIf { false } }
    }
  }
}

reporting {
  reports {
    testAggregateTestReport(AggregateTestReport) {
      testType = TestSuiteType.UNIT_TEST
    }
  }
}

// Enable XML test reports for Jenkins integration
tasks.withType(TestTaskReports).configureEach {
  junitXml.enabled = true
}

tasks.named(JavaBasePlugin.CHECK_TASK_NAME) {
  dependsOn tasks.named('testAggregateTestReport', TestReport)
}