build.sbt

import java.nio.file.Files
import java.io.File
import Tarball.createTarballSettings
import sbt.util
import sbtlicensereport.license.{DepModuleInfo, LicenseCategory, LicenseInfo}
import ReleaseSettings.{javaOnlyReleaseSettings, rootReleaseSettings, skipReleaseSettings}

import scala.language.implicitConversions

val orgName = "io.unitycatalog"
val artifactNamePrefix = "unitycatalog"

// Use Java 11 for two modules: clients and spark
// for better Spark compatibility
// until Spark 4 comes out with newer Java compatibility
lazy val javacRelease11 = Seq("--release", "11")
lazy val javacRelease17 = Seq("--release", "17")

lazy val scala212 = "2.12.15"
lazy val scala213 = "2.13.14"

lazy val commonSettings = Seq(
  organization := orgName,
  // Compilation configs
  initialize := {
    // Assert that the JVM is at least Java 17
    val _ = initialize.value  // ensure previous initializations are run
    assert(
      sys.props("java.specification.version").toDouble >= 17,
      "Java 17 or above is required to run this project.")
  },
  Compile / compile / javacOptions ++= Seq(
    "-Xlint:deprecation",
    "-Xlint:unchecked",
    "-g:source,lines,vars",
  ),
  Test / javaOptions ++= Seq (
    "-ea",
  ),
  libraryDependencies ++= Seq(
    "org.slf4j" % "slf4j-api" % "2.0.13",
    "org.slf4j" % "slf4j-log4j12" % "2.0.13" % Test,
    "org.apache.logging.log4j" % "log4j-slf4j2-impl" % "2.23.1",
    "org.apache.logging.log4j" % "log4j-api" % "2.23.1"
  ),
  resolvers += Resolver.mavenLocal,
  autoScalaLibrary := false,
  crossPaths := false,  // No scala cross building
  assembly / assemblyMergeStrategy := {
    case PathList("META-INF", xs @ _*) => MergeStrategy.discard
    case _ => MergeStrategy.first
  },

  // Test configs
  Test / testOptions  := Seq(Tests.Argument(TestFrameworks.JUnit, "-a", "-v", "-q"), Tests.Filter(name => !(name startsWith s"$orgName.server.base"))),
  Test / logLevel := util.Level.Info,
  Test / publishArtifact := false,
  fork := true,
  outputStrategy := Some(StdoutOutput),

  Compile / packageBin := {
    val packageFile = (Compile / packageBin).value
    generateClasspathFile(
      targetDir = packageFile.getParentFile,
      classpath = (Runtime / dependencyClasspath).value)
    packageFile
  },

  licenseConfigurations := Set("compile"),
  licenseOverrides := {
    case DepModuleInfo("io.unitycatalog", _, _) =>
      LicenseInfo(LicenseCategory.Apache, "Apache 2.0", "http://www.apache.org/licenses")
    case DepModuleInfo("org.hibernate.common", "hibernate-commons-annotations", _) =>
      // Apache 2.0: https://mvnrepository.com/artifact/org.hibernate.common/hibernate-commons-annotations
      LicenseInfo(LicenseCategory.Apache, "Apache 2.0", "http://www.apache.org/licenses")
  },
  licenseDepExclusions := {
    // LGPL 2.1: https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html
    // https://en.wikipedia.org/wiki/GNU_Lesser_General_Public_License
    // We can use and distribute the, but not modify the source code
    case DepModuleInfo("org.hibernate.orm", _, _) => true
    // Duo license:
    //  - Eclipse Public License 2.0
    //  - GNU General Public License, version 2 with the GNU Classpath Exception
    // I think we're good with the classpath exception in there.
    case DepModuleInfo("jakarta.transaction", "jakarta.transaction-api", _) => true
    case DepModuleInfo("javax.annotation", "javax.annotation-api", _) => true
  },
  
  assembly / test := {}
)

enablePlugins(CoursierPlugin)

useCoursier := true

// Configure resolvers
resolvers ++= Seq(
  "Sonatype OSS Releases" at "https://oss.sonatype.org/content/repositories/releases/",
  "Maven Central" at "https://repo1.maven.org/maven2/",
)

def javaCheckstyleSettings(configLocation: File) = Seq(
  checkstyleConfigLocation := CheckstyleConfigLocation.File(configLocation.toString),
  checkstyleSeverityLevel := Some(CheckstyleSeverityLevel.Error),
  // (Compile / compile) := ((Compile / compile) dependsOn (Compile / checkstyle)).value,
  // (Test / test) := ((Test / test) dependsOn (Test / checkstyle)).value,
)

// enforce java code style
def javafmtCheckSettings() = Seq(
  (Compile / compile) := ((Compile / compile) dependsOn (Compile / javafmtCheckAll)).value
)

lazy val client = (project in file("target/clients/java"))
  .enablePlugins(OpenApiGeneratorPlugin)
  .disablePlugins(JavaFormatterPlugin)
  .settings(
    name := s"$artifactNamePrefix-client",
    commonSettings,
    javaOnlyReleaseSettings,
    Compile / compile / javacOptions ++= javacRelease11,
    libraryDependencies ++= Seq(
      "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
      "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion,
      "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion,
      "org.openapitools" % "jackson-databind-nullable" % openApiToolsJacksonBindNullableVersion,
      "com.google.code.findbugs" % "jsr305" % "3.0.2",
      "jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,

      // Test dependencies
      "org.junit.jupiter" % "junit-jupiter" % "5.10.3" % Test,
      "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test,
      "org.assertj" % "assertj-core" % "3.26.3" % Test,
    ),
    (Compile / compile) := ((Compile / compile) dependsOn generate).value,

    // OpenAPI generation specs
    openApiInputSpec := (file(".") / "api" / "all.yaml").toString,
    openApiGeneratorName := "java",
    openApiOutputDir := (file("target") / "clients" / "java").toString,
    openApiApiPackage := s"$orgName.client.api",
    openApiModelPackage := s"$orgName.client.model",
    openApiAdditionalProperties := Map(
      "library" -> "native",
      "useJakartaEe" -> "true",
      "hideGenerationTimestamp" -> "true",
      "openApiNullable" -> "false"),
    openApiGenerateApiTests := SettingDisabled,
    openApiGenerateModelTests := SettingDisabled,
    openApiGenerateApiDocumentation := SettingDisabled,
    openApiGenerateModelDocumentation := SettingDisabled,
    // Define the simple generate command to generate full client codes
    generate := {
      val _ = openApiGenerate.value

      // Delete the generated build.sbt file so that it is not used for our sbt config
      val buildSbtFile = file(openApiOutputDir.value) / "build.sbt"
      if (buildSbtFile.exists()) {
        buildSbtFile.delete()
      }
    }
  )

lazy val apiDocs = (project in file("api"))
  .enablePlugins(OpenApiGeneratorPlugin)
  .settings(
    name := s"$artifactNamePrefix-docs",

    // OpenAPI generation specs
    openApiInputSpec := (file("api") / "all.yaml").toString,
    openApiGeneratorName := "markdown",
    openApiOutputDir := (file("api")).toString,
    // Define the simple generate command to generate markdown docs
    generate := {
      val _ = openApiGenerate.value
    }
  )

// Define the custom task key
lazy val populateTestDB = taskKey[Unit]("Run PopulateTestDatabase main class from the test folder")

lazy val server = (project in file("server"))
  .dependsOn(client % "test->test")
  .dependsOn(serverModels)
  .settings (
    name := s"$artifactNamePrefix-server",
    mainClass := Some(orgName + ".server.UnityCatalogServer"),
    commonSettings,
    javaOnlyReleaseSettings,
    javafmtCheckSettings,
    javaCheckstyleSettings(file("dev") / "checkstyle-config.xml"),
    Compile / compile / javacOptions ++= Seq(
      "-processor",
      "lombok.launch.AnnotationProcessorHider$AnnotationProcessor"
    ) ++ javacRelease17,
    libraryDependencies ++= Seq(
      "com.linecorp.armeria" %  "armeria" % "1.28.4",
      // Netty dependencies
      "io.netty" % "netty-all" % "4.1.111.Final",
      "jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,
      // Jackson dependencies
      "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
      "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion,
      "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
      "com.fasterxml.jackson.dataformat" % "jackson-dataformat-yaml" % jacksonVersion,
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion,
      "com.auth0" % "java-jwt" % "4.4.0",
      "com.auth0" % "jwks-rsa" % "0.22.1",

      "com.google.code.findbugs" % "jsr305" % "3.0.2",
      "com.h2database" %  "h2" % "2.2.224",

      "org.hibernate.orm" % "hibernate-core" % "6.5.0.Final",

      "jakarta.activation" % "jakarta.activation-api" % "2.1.3",
      "net.bytebuddy" % "byte-buddy" % "1.14.15",
      "org.projectlombok" % "lombok" % "1.18.32" % Provided,

      // For ALDS access
      "com.azure" % "azure-identity" % "1.13.2",
      "com.azure" % "azure-storage-file-datalake" % "12.20.0",

      // For GCS Access
      "com.google.cloud" % "google-cloud-storage" % "2.30.1",
      "com.google.auth" % "google-auth-library-oauth2-http" % "1.20.0",

      //For s3 access
      "com.amazonaws" % "aws-java-sdk-s3" % "1.12.728",
      "org.apache.httpcomponents" % "httpcore" % "4.4.16",
      "org.apache.httpcomponents" % "httpclient" % "4.5.14",

      // Iceberg REST Catalog dependencies
      "org.apache.iceberg" % "iceberg-core" % "1.5.2",
      "org.apache.iceberg" % "iceberg-aws" % "1.5.2",
      "org.apache.iceberg" % "iceberg-azure" % "1.5.2",
      "org.apache.iceberg" % "iceberg-gcp" % "1.5.2",
      "software.amazon.awssdk" % "s3" % "2.24.0",
      "software.amazon.awssdk" % "sts" % "2.24.0",
      "io.vertx" % "vertx-core" % "4.3.5",
      "io.vertx" % "vertx-web" % "4.3.5",
      "io.vertx" % "vertx-web-client" % "4.3.5",

      // Test dependencies
      "org.junit.jupiter" %  "junit-jupiter" % "5.10.3" % Test,
      "org.mockito" % "mockito-core" % "5.11.0" % Test,
      "org.mockito" % "mockito-inline" % "5.2.0" % Test,
      "org.mockito" % "mockito-junit-jupiter" % "5.12.0" % Test,
      "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test,
      "com.adobe.testing" % "s3mock-junit5" % "3.9.1" % Test
        exclude("ch.qos.logback", "logback-classic")
        exclude("org.apache.logging.log4j", "log4j-to-slf4j"),
      "javax.xml.bind" % "jaxb-api" % "2.3.1" % Test,

      // CLI dependencies
      "commons-cli" % "commons-cli" % "1.7.0"
    ),

    Compile / sourceGenerators += Def.task {
      val file = (Compile / sourceManaged).value / "io" / "unitycatalog" / "server" / "utils" / "VersionUtils.java"
      IO.write(file,
        s"""package io.unitycatalog.server.utils;
           |
           |public class VersionUtils {
           |  public static String VERSION = "${version.value}";
           |}
           |""".stripMargin)
      Seq(file)
    },
    populateTestDB := {
      val log = streams.value.log
      (Test / runMain).toTask(s" io.unitycatalog.server.utils.PopulateTestDatabase").value
    },
    Test / javaOptions += s"-Duser.dir=${(ThisBuild / baseDirectory).value.getAbsolutePath}"
  )

lazy val serverModels = (project in file("server") / "target" / "models")
  .enablePlugins(OpenApiGeneratorPlugin)
  .disablePlugins(JavaFormatterPlugin)
  .settings(
    name := s"$artifactNamePrefix-servermodels",
    commonSettings,
    (Compile / compile) := ((Compile / compile) dependsOn generate).value,
    Compile / compile / javacOptions ++= javacRelease17,
    libraryDependencies ++= Seq(
      "jakarta.annotation" % "jakarta.annotation-api" % "3.0.0" % Provided,
      "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion,
    ),
    // OpenAPI generation configs for generating model codes from the spec
    openApiInputSpec := (file(".") / "api" / "all.yaml").toString,
    openApiGeneratorName := "java",
    openApiOutputDir := (file("server") / "target" / "models").toString,
    openApiValidateSpec := SettingEnabled,
    openApiGenerateMetadata := SettingDisabled,
    openApiModelPackage := s"$orgName.server.model",
    openApiAdditionalProperties := Map(
      "library" -> "resteasy", // resteasy generates the most minimal models
      "useJakartaEe" -> "true",
      "hideGenerationTimestamp" -> "true"
    ),
    openApiGlobalProperties := Map("models" -> ""),
    openApiGenerateApiTests := SettingDisabled,
    openApiGenerateModelTests := SettingDisabled,
    openApiGenerateApiDocumentation := SettingDisabled,
    openApiGenerateModelDocumentation := SettingDisabled,
    // Define the simple generate command to generate model codes
    generate := {
      val _ = openApiGenerate.value
    }
  )

lazy val cli = (project in file("examples") / "cli")
  .dependsOn(server % "test->test")
  .dependsOn(client % "compile->compile;test->test")
  .settings(
    name := s"$artifactNamePrefix-cli",
    mainClass := Some(orgName + ".cli.UnityCatalogCli"),
    commonSettings,
    skipReleaseSettings,
    javafmtCheckSettings,
    javaCheckstyleSettings(file("dev") / "checkstyle-config.xml"),
    Compile / compile / javacOptions ++= javacRelease17,
    libraryDependencies ++= Seq(
      "commons-cli" % "commons-cli" % "1.7.0",
      "org.json" % "json" % "20240303",
      "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion,
      "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % jacksonVersion,
      "org.openapitools" % "jackson-databind-nullable" % openApiToolsJacksonBindNullableVersion,
      "org.yaml" % "snakeyaml" % "2.2",

      "io.delta" % "delta-kernel-api" % "3.2.0",
      "io.delta" % "delta-kernel-defaults" % "3.2.0",
      "io.delta" % "delta-storage" % "3.2.0",
      "org.apache.hadoop" % "hadoop-client-api" % "3.4.0",
      "org.apache.hadoop" % "hadoop-client-runtime" % "3.4.0",
      "de.vandermeer" % "asciitable" % "0.3.2",
      // for s3 access
      "org.fusesource.jansi" % "jansi" % "2.4.1",
      "com.amazonaws" % "aws-java-sdk-core" % "1.12.728",
      "org.apache.hadoop" % "hadoop-aws" % "3.4.0",
      "com.google.guava" % "guava" % "31.0.1-jre",
      // Test dependencies
      "org.junit.jupiter" % "junit-jupiter" % "5.10.3" % Test,
      "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test,
    ),
    Test / javaOptions += s"-Duser.dir=${(ThisBuild / baseDirectory).value.getAbsolutePath}",
  )

/*
  * This project is a combination of the server and client projects, shaded into a single JAR.
  * It also includes the test classes from the server project.
  * It is used for the Spark connector project(the client is required as a compile dependency,
  * and the server(with tests) is required as a test dependency)
  * This was necessary because Spark 3.5 has a dependency on Jackson 2.15, which conflicts with the Jackson 2.17
 */
lazy val serverShaded = (project in file("server-shaded"))
  .dependsOn(server % "compile->compile, test->compile")
  .settings(
    name := s"$artifactNamePrefix-server-shaded",
    commonSettings,
    skipReleaseSettings,
    Compile / packageBin := assembly.value,
    assembly / mainClass := Some("io.unitycatalog.server.UnityCatalogServer"),
    assembly / logLevel := Level.Warn,
    assembly / test := {},
    assembly / assemblyShadeRules := Seq(
      ShadeRule.rename("com.fasterxml.**" -> "shaded.@0").inAll,
      ShadeRule.rename("org.antlr.**" -> "shaded.@0").inAll,
    ),
    assemblyPackageScala / assembleArtifact := false,
    assembly / fullClasspath := {
      val compileClasspath = (server / Compile / fullClasspath).value
      val testClasses = (server / Test / products).value
      compileClasspath ++ testClasses.map(Attributed.blank)
    }
  )

val sparkVersion = "3.5.1"
lazy val spark = (project in file("connectors/spark"))
  .dependsOn(client)
  .settings(
    name := s"$artifactNamePrefix-spark",
    scalaVersion := scala212,
    crossScalaVersions := Seq(scala212, scala213),
    commonSettings,
    javaOptions ++= Seq(
      "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
    ),
    javaCheckstyleSettings(file("dev/checkstyle-config.xml")),
    Compile / compile / javacOptions ++= javacRelease11,
    libraryDependencies ++= Seq(
      "org.apache.spark" %% "spark-sql" % sparkVersion,
      "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.0",
      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.0",
      "com.fasterxml.jackson.core" % "jackson-annotations" % "2.15.0",
      "com.fasterxml.jackson.core" % "jackson-core" % "2.15.0",
      "com.fasterxml.jackson.dataformat" % "jackson-dataformat-xml" % "2.15.0",
      "org.antlr" % "antlr4-runtime" % "4.9.3",
      "org.antlr" % "antlr4" % "4.9.3",
      "com.google.cloud.bigdataoss" % "util-hadoop" % "3.0.2" % Provided,
    ),
    libraryDependencies ++= Seq(
      // Test dependencies
      "org.junit.jupiter" % "junit-jupiter" % "5.10.3" % Test,
      "org.assertj" % "assertj-core" % "3.26.3" % Test,
      "org.mockito" % "mockito-core" % "5.11.0" % Test,
      "org.mockito" % "mockito-inline" % "5.2.0" % Test,
      "org.mockito" % "mockito-junit-jupiter" % "5.12.0" % Test,
      "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test,
      "org.apache.hadoop" % "hadoop-client-runtime" % "3.4.0",
      "io.delta" %% "delta-spark" % "3.2.0" % Test,
    ),
    dependencyOverrides ++= Seq(
      "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.0",
      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.15.0",
      "com.fasterxml.jackson.core" % "jackson-annotations" % "2.15.0",
      "com.fasterxml.jackson.core" % "jackson-core" % "2.15.0",
      "com.fasterxml.jackson.dataformat" % "jackson-dataformat-xml" % "2.15.0",
      "org.antlr" % "antlr4-runtime" % "4.9.3",
      "org.antlr" % "antlr4" % "4.9.3",
    ),
    Test / unmanagedJars += (serverShaded / assembly).value,
    licenseDepExclusions := {
      case DepModuleInfo("org.hibernate.orm", _, _) => true
      case DepModuleInfo("jakarta.annotation", "jakarta.annotation-api", _) => true
      case DepModuleInfo("jakarta.servlet", "jakarta.servlet-api", _) => true
      case DepModuleInfo("jakarta.transaction", "jakarta.transaction-api", _) => true
      case DepModuleInfo("jakarta.ws.rs", "jakarta.ws.rs-api", _) => true
      case DepModuleInfo("javax.activation", "activation", _) => true
      case DepModuleInfo("javax.servlet", "javax.servlet-api", _) => true
      case DepModuleInfo("org.glassfish.hk2", "hk2-api", _) => true
      case DepModuleInfo("org.glassfish.hk2", "hk2-locator", _) => true
      case DepModuleInfo("org.glassfish.hk2", "hk2-utils", _) => true
      case DepModuleInfo("org.glassfish.hk2", "osgi-resource-locator", _) => true
      case DepModuleInfo("org.glassfish.hk2.external", "aopalliance-repackaged", _) => true
      case DepModuleInfo("ch.qos.logback", "logback-classic", _) => true
      case DepModuleInfo("ch.qos.logback", "logback-core", _) => true
      case DepModuleInfo("org.apache.xbean", "xbean-asm9-shaded", _) => true
      case DepModuleInfo("oro", "oro", _) => true
      case DepModuleInfo("org.glassfish", "javax.json", _) => true
      case DepModuleInfo("org.glassfish.hk2.external", "jakarta.inject", _) => true
      case DepModuleInfo("org.antlr", "ST4", _) => true
    }
  )

lazy val root = (project in file("."))
  .aggregate(serverModels, client, server, cli, spark)
  .settings(
    name := s"$artifactNamePrefix",
    createTarballSettings(),
    commonSettings,
    rootReleaseSettings
  )

def generateClasspathFile(targetDir: File, classpath: Classpath): Unit = {
  // Generate a classpath file with the entire runtime class path.
  // This is used by the launcher scripts for launching CLI directly with JAR instead of SBT.
  val classpathFile = targetDir / "classpath"
  Files.write(classpathFile.toPath, classpath.files.mkString(File.pathSeparator).getBytes)
  println(s"Generated classpath file '$classpathFile'")
}

val generate = taskKey[Unit]("generate code from APIs")

// Library versions
val jacksonVersion = "2.17.0"
val openApiToolsJacksonBindNullableVersion = "0.2.6"
val log4jVersion = "2.23.1"