Merge branch '2000-upgrade-emr' into 3489-upgrade-java

java/ingest/ingest-runner/src/main/java/sleeper/ingest/impl/recordbatch/arrow/ArrowIngestSupport.java

@@ -20,12 +20,12 @@
 import org.apache.arrow.algorithm.sort.IndexSorter;
 import org.apache.arrow.algorithm.sort.VectorValueComparator;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BaseVariableWidthVector;
 import org.apache.arrow.vector.BigIntVector;
 import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
-import org.apache.arrow.vector.VariableWidthVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 
 import sleeper.core.schema.type.ByteArrayType;
@@ -82,11 +82,11 @@ public static IntVector createSortOrderVector(BufferAllocator bufferAllocator,
  vectorValueComparator.attachVector((BigIntVector) vectorSchemaRoot.getVector(indexOfField));
  return vectorValueComparator;
  } else if (fieldType instanceof StringType) {
- VectorValueComparator<VariableWidthVector> vectorValueComparator = new DefaultVectorComparators.VariableWidthComparator();
+ VectorValueComparator<BaseVariableWidthVector> vectorValueComparator = new DefaultVectorComparators.VariableWidthComparator();
  vectorValueComparator.attachVector((VarCharVector) vectorSchemaRoot.getVector(indexOfField));
  return vectorValueComparator;
  } else if (fieldType instanceof ByteArrayType) {
- VectorValueComparator<VariableWidthVector> vectorValueComparator = new DefaultVectorComparators.VariableWidthComparator();
+ VectorValueComparator<BaseVariableWidthVector> vectorValueComparator = new DefaultVectorComparators.VariableWidthComparator();
  vectorValueComparator.attachVector((VarBinaryVector) vectorSchemaRoot.getVector(indexOfField));
  return vectorValueComparator;
  } else {

java/pom.xml

@@ -67,6 +67,8 @@
  <hadoop.version>3.3.6</hadoop.version>
  <parquet-hadoop.version>1.14.1</parquet-hadoop.version>
  <joom.spark.platform.version>0.4.7</joom.spark.platform.version>
+ <!-- The version of Arrow should match the version used by Spark -->
+ <arrow.version>12.0.1</arrow.version>
  <!--
  Spark Catalyst uses an old version of DataSketches. This must match the version declared there, as it is
  provided in EMR.
@@ -91,23 +93,21 @@
  <snappy.version>1.1.10.7</snappy.version>
  <!-- Parquet declares an old version of Aircompressor with vulnerabilities. Managed from 0.21. -->
  <aircompressor.version>0.27</aircompressor.version>
- <!-- Hadoop declares an old version of Protobuf with vulnerabilities. Managed from 2.5.0. -->
+ <!-- Hadoop declares an old version of Protobuf with vulnerabilities. Managed from 3.23.4. -->
  <protobuf.version>4.28.2</protobuf.version>
  <!-- Hadoop declares an old version of Nimbus with vulnerabilities. Managed from 9.8.1. -->
  <nimbus-jose-jwt.version>9.41.2</nimbus-jose-jwt.version>
- <!-- Hadoop declares an old version of Avro with vulnerabilities. Managed from 1.7.7. -->
+ <!-- Spark declares an old version of Avro with vulnerabilities. Managed from 1.11.2. -->
  <avro.version>1.12.0</avro.version>
- <!-- Hadoop declares an old version of json-smart with vulnerabilities. Managed from 2.4.7. -->
- <json-smart.version>2.5.1</json-smart.version>
  <!-- Hadoop declares an old version of Kerby with vulnerabilities. Managed from 1.0.1. -->
  <kerby.version>2.1.0</kerby.version>
  <!-- Hadoop declares an old version of XNIO with vulnerabilities. Managed from 3.8.8.Final. -->
  <xnio.version>3.8.16.Final</xnio.version>
  <!-- Hadoop declares an old version of dnsjava with vulnerabilities. Managed from 2.1.7. -->
  <dnsjava.version>3.6.2</dnsjava.version>
- <!-- Spark declares an old version of XZ with vulnerabilities. Managed from 1.9. -->
+ <!-- Spark (via Avro) declares an old version of XZ with vulnerabilities. Managed from 1.9. -->
  <xz.version>1.10</xz.version>
- <!-- Spark declares an old version of OkHttp with vulnerabilities. Managed from 4.10.0. -->
+ <!-- Spark Platform declares an old version of OkHttp with vulnerabilities. Managed from 4.10.0. -->
  <okhttp3.version>4.12.0</okhttp3.version>
  <!--
  OkHttp, as a dependency of Spark, declares an old version of Kotlin with vulnerabilities.
@@ -123,7 +123,7 @@
  <!-- jungrapht-visualization-samples also declares an old version, which is a dependency of the build module. -->
  <logback.version>1.5.11</logback.version>
  <aws-java-sdk.version>1.12.498</aws-java-sdk.version>
- <aws-java-sdk-v2.version>2.28.23</aws-java-sdk-v2.version>
+ <aws-java-sdk-v2.version>2.28.24</aws-java-sdk-v2.version>
  <aws-crt.version>0.31.3</aws-crt.version>
  <aws-lambda-java-events.version>3.14.0</aws-lambda-java-events.version>
  <aws-lambda-java-core.version>1.2.3</aws-lambda-java-core.version>
@@ -151,7 +151,6 @@
  <slf4j.version>2.0.16</slf4j.version>
  <reload4j.version>1.2.25</reload4j.version>
  <java-websocket.version>1.5.7</java-websocket.version>
- <arrow.version>17.0.0</arrow.version>
  <bouncycastle.version>1.78.1</bouncycastle.version>
  <athena.version>2023.3.1</athena.version>
  <trino.version>390</trino.version>
@@ -647,11 +646,6 @@
  <artifactId>avro-mapred</artifactId>
  <version>${avro.version}</version>
  </dependency>
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
  <dependency>
  <groupId>org.apache.kerby</groupId>
  <artifactId>kerb-core</artifactId>

rust/rust_sketch/Cargo.toml

@@ -31,7 +31,7 @@ git2 = { version = "0.19.0" }
 
 [target.'cfg(target_os = "macos")'.build-dependencies]
 # Mac often won't have openssl library installed in place easy discoverable, if at all
-openssl = { version = '0.10.67', features = [
+openssl = { version = '0.10.68', features = [
  "vendored",
 ] } # Use vendored feature to build from source for cross compilation