Release bump 2 4 0 (#63)

* NOSNOW: added utility to upload connector to our nexus for simplified k8s deployment (snowflakedb#874) (cherry picked from commit ee83e22) * SNOW-1514185: Do assign the reopened channel unless kafka offsets are fully reset (snowflakedb#875) (cherry picked from commit 05dcbdf) * SNOW-1061851: Run SnowflakeSinkServiceV2IT for a single buffer (snowflakedb#876) (cherry picked from commit 7fce0f5) * SNOW-1061855 Fix the E2E test for SchemaEvolutionDropTable with a single buffer (snowflakedb#882) (cherry picked from commit fbd15d6) * NOSNOW: improve troubleshooting experience for scenario with skipped offsets (snowflakedb#883) (cherry picked from commit 7172481) * Adding logs to track schematization (snowflakedb#884) (cherry picked from commit 9905c7f) * Remove insertRows DEBUG log (called once per row) (snowflakedb#886) (cherry picked from commit 363457d) * NO-SNOW Upgrade nexus-staging-maven-plugin version (snowflakedb#887) (cherry picked from commit ac1276e) * Upgrade com.google.protobuf to 3.24.4 (snowflakedb#892) (cherry picked from commit 3c7e18c) * SNOW-1229110 Add single buffer usage data to telemetry (snowflakedb#893) (cherry picked from commit 96d3bfa) * SNOW-1541942 Extend Snowpipe initialization logging at INFO level (snowflakedb#885) (cherry picked from commit eab4c9c) * SNOW-947731 Remove deprecated avro-python3 package (snowflakedb#889) (cherry picked from commit 47624f5) * Bump dev.failsafe:failsafe from 3.2.1 to 3.3.2 (snowflakedb#859) Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michał Bobowski <[email protected]> (cherry picked from commit 4a22f56) * Bump org.assertj:assertj-core from 3.25.3 to 3.26.3 (snowflakedb#881) Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michał Bobowski <[email protected]> (cherry picked from commit adb896c) * Bump io.dropwizard.metrics:metrics-core from 4.2.3 to 4.2.26 (snowflakedb#863) Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Michał Bobowski <[email protected]> (cherry picked from commit 258ed2d) * NO-SNOW Ignore OAuth test (snowflakedb#906) (cherry picked from commit 30d2e8b) * SNOW-1229108 Add warning about buffer config when single buffer enabled (snowflakedb#896) (cherry picked from commit 9aa0b0b) * NO_SNOW Update dependencies (snowflakedb#898) Co-authored-by: Michał Bobowski <[email protected]> (cherry picked from commit eff23e2) * SNOW-1618257 - upgrade to 2.2.0 ingest-sdk (snowflakedb#910) (cherry picked from commit d0e97fb) * NO-SNOW Release v.2.4.0 (snowflakedb#907) Co-authored-by: Xin Huang <[email protected]> (cherry picked from commit 05c1148) --------- Co-authored-by: Greg Jachimko <[email protected]> Co-authored-by: Adrian Kowalczyk <[email protected]> Co-authored-by: Michał Bobowski <[email protected]> Co-authored-by: Xin Huang <[email protected]> Co-authored-by: Wojciech Trefon <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
confluentinc · Aug 22, 2024 · f1067de · f1067de
1 parent cb10a97
commit f1067de
Show file tree

Hide file tree

Showing 40 changed files with 1,598 additions and 1,088 deletions.
diff --git a/.github/workflows/End2EndTestApache.yml b/.github/workflows/End2EndTestApache.yml
@@ -43,9 +43,9 @@ jobs:
  run: |
  pip3 install --upgrade setuptools
  sudo apt-get install librdkafka-dev
- pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0"
- pip3 install avro-python3 kafka-python
- pip3 install --upgrade protobuf==3.20.0
+ pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2"
+ pip3 install avro kafka-python
+ pip3 install --upgrade protobuf==3.20.3
  pip3 install --upgrade snowflake-connector-python==2.7.4
  curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
  sudo apt-get -y install jq vim

diff --git a/.github/workflows/End2EndTestConfluent.yml b/.github/workflows/End2EndTestConfluent.yml
@@ -43,9 +43,9 @@ jobs:
  run: |
  pip3 install --upgrade setuptools
  sudo apt-get install librdkafka-dev
- pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0"
- pip3 install avro-python3 kafka-python
- pip3 install --upgrade protobuf==3.20.0
+ pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2"
+ pip3 install avro kafka-python
+ pip3 install --upgrade protobuf==3.20.3
  pip3 install --upgrade snowflake-connector-python==2.7.4
  curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
  sudo apt-get -y install jq vim

diff --git a/.github/workflows/IntegrationTest.yml b/.github/workflows/IntegrationTest.yml
@@ -39,8 +39,8 @@ jobs:
  - name: Install Dependency
  run: |
  pip3 install --upgrade setuptools
- pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0"
- pip3 install avro-python3 kafka-python
+ pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2"
+ pip3 install avro kafka-python
  pip3 install protobuf
  pip3 install --upgrade snowflake-connector-python==2.7.4
  curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash

diff --git a/.github/workflows/StressTest.yml b/.github/workflows/StressTest.yml
@@ -36,9 +36,9 @@ jobs:
  run: |
  pip3 install --upgrade setuptools
  sudo apt-get install librdkafka-dev
- pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0"
- pip3 install avro-python3 kafka-python
- pip3 install --upgrade protobuf==3.20.0
+ pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.9.2"
+ pip3 install avro kafka-python
+ pip3 install --upgrade protobuf==3.20.3
  pip3 install --upgrade snowflake-connector-python==2.7.4
  curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
  sudo apt-get -y install jq

diff --git a/pom.xml b/pom.xml
@@ -12,7 +12,7 @@
 
  <groupId>com.snowflake</groupId>
  <artifactId>snowflake-kafka-connector</artifactId>
- <version>2.3.0</version>
+ <version>2.4.0</version>
  <packaging>jar</packaging>
  <name>Snowflake Kafka Connector</name>
  <description>Snowflake Kafka Connect Sink Connector</description>
@@ -52,8 +52,10 @@
 
  <kafka.version>3.7.0</kafka.version>
  <awaitility.version>4.0.0</awaitility.version>
- <assertj-core.version>3.25.3</assertj-core.version>
- <confluent.version>7.6.0</confluent.version>
+ <assertj-core.version>3.26.3</assertj-core.version>
+ <confluent.version>7.7.0</confluent.version>
+ <!--Compatible protobuf version https:/confluentinc/common/blob/v7.7.0/pom.xml#L91 -->
+ <protobuf.version>3.24.4</protobuf.version>
  </properties>
 
 
@@ -147,7 +149,7 @@
  <plugin>
  <groupId>org.sonatype.plugins</groupId>
  <artifactId>nexus-staging-maven-plugin</artifactId>
- <version>1.6.7</version>
+ <version>1.7.0</version>
  <extensions>true</extensions>
  <configuration>
  <serverId>ossrh</serverId>
@@ -352,14 +354,14 @@
  <dependency>
  <groupId>net.snowflake</groupId>
  <artifactId>snowflake-jdbc</artifactId>
- <version>3.14.5</version>
+ <version>3.18.0</version>
  </dependency>
 
  <!-- Ingest SDK for copy staged file into snowflake table -->
  <dependency>
  <groupId>net.snowflake</groupId>
  <artifactId>snowflake-ingest-sdk</artifactId>
- <version>2.1.1</version>
+ <version>2.2.0</version>
  <exclusions>
  <exclusion>
  <groupId>net.snowflake</groupId>
@@ -398,18 +400,18 @@
  <artifactId>commons-compress</artifactId>
  <version>1.26.0</version>
  </dependency>
-
+ 
  <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
  <dependency>
  <groupId>com.fasterxml.jackson.core</groupId>
  <artifactId>jackson-core</artifactId>
- <version>2.16.1</version>
- </dependency>
+ <version>2.17.2</version>
+ </dependency> 
  <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
  <dependency>
  <groupId>com.fasterxml.jackson.core</groupId>
  <artifactId>jackson-databind</artifactId>
- <version>2.16.1</version>
+ <version>2.17.2</version>
  </dependency>
 
  <!-- <https://mvnrepository.com/artifact/io.confluent/kafka-schema-registry-client?repo=confluent-packages -->
@@ -441,7 +443,6 @@
  <groupId>io.confluent</groupId>
  <artifactId>kafka-avro-serializer</artifactId>
  <version>${confluent.version}</version>
- <scope>provided</scope>
  <exclusions>
  <exclusion>
  <groupId>io.confluent</groupId>
@@ -454,7 +455,6 @@
  <groupId>io.confluent</groupId>
  <artifactId>kafka-connect-avro-converter</artifactId>
  <version>${confluent.version}</version>
- <scope>provided</scope>
  </dependency>
 
  <dependency>
@@ -473,7 +473,7 @@
  <dependency>
  <groupId>io.dropwizard.metrics</groupId>
  <artifactId>metrics-core</artifactId>
- <version>4.2.3</version>
+ <version>4.2.26</version>
  </dependency>
  <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
  <dependency>
@@ -488,11 +488,22 @@
  <version>32.0.1-jre</version>
  </dependency>
 
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+
  <!-- https:/failsafe-lib/failsafe-->
  <dependency>
  <groupId>dev.failsafe</groupId>
  <artifactId>failsafe</artifactId>
- <version>3.2.1</version>
+ <version>3.3.2</version>
  </dependency>
 
  <!--junit for unit test-->

diff --git a/pom_confluent.xml b/pom_confluent.xml
@@ -12,7 +12,7 @@
 
  <groupId>com.snowflake</groupId>
  <artifactId>snowflake-kafka-connector</artifactId>
- <version>2.3.0</version>
+ <version>2.4.0</version>
  <packaging>jar</packaging>
  <name>Snowflake Kafka Connector</name>
  <description>Snowflake Kafka Connect Sink Connector</description>
@@ -62,8 +62,10 @@
 
  <kafka.version>3.7.0</kafka.version>
  <awaitility.version>4.0.0</awaitility.version>
- <assertj-core.version>3.25.3</assertj-core.version>
- <confluent.version>7.6.0</confluent.version>
+ <assertj-core.version>3.26.3</assertj-core.version>
+ <confluent.version>7.7.0</confluent.version>
+ <!--Compatible protobuf version https:/confluentinc/common/blob/v7.7.0/pom.xml#L91 -->
+ <protobuf.version>3.24.4</protobuf.version>
  </properties>
 
 
@@ -305,7 +307,7 @@
  <plugin>
  <groupId>org.sonatype.plugins</groupId>
  <artifactId>nexus-staging-maven-plugin</artifactId>
- <version>1.6.7</version>
+ <version>1.7.0</version>
  <extensions>true</extensions>
  <configuration>
  <serverId>ossrh</serverId>
@@ -482,14 +484,25 @@
  <dependency>
  <groupId>net.snowflake</groupId>
  <artifactId>snowflake-jdbc</artifactId>
- <version>3.14.5</version>
+ <version>3.18.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java-util</artifactId>
+ <version>${protobuf.version}</version>
  </dependency>
 
  <!-- Ingest SDK for copy staged file into snowflake table -->
  <dependency>
  <groupId>net.snowflake</groupId>
  <artifactId>snowflake-ingest-sdk</artifactId>
- <version>2.1.1</version>
+ <version>2.2.0</version>
  <exclusions>
  <exclusion>
  <groupId>net.snowflake</groupId>
@@ -549,13 +562,13 @@
  <dependency>
  <groupId>com.fasterxml.jackson.core</groupId>
  <artifactId>jackson-core</artifactId>
- <version>2.16.1</version>
+ <version>2.17.2</version>
  </dependency>
  <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
  <dependency>
  <groupId>com.fasterxml.jackson.core</groupId>
  <artifactId>jackson-databind</artifactId>
- <version>2.16.1</version>
+ <version>2.17.2</version>
  </dependency>
 
  <!-- <https://mvnrepository.com/artifact/io.confluent/kafka-schema-registry-client?repo=confluent-packages -->
@@ -619,7 +632,7 @@
  <dependency>
  <groupId>io.dropwizard.metrics</groupId>
  <artifactId>metrics-core</artifactId>
- <version>4.2.3</version>
+ <version>4.2.26</version>
  </dependency>
  <!-- https://mvnrepository.com/artifact/io.dropwizard.metrics/metrics-jmx -->
  <dependency>
@@ -638,7 +651,7 @@
  <dependency>
  <groupId>dev.failsafe</groupId>
  <artifactId>failsafe</artifactId>
- <version>3.2.1</version>
+ <version>3.3.2</version>
  </dependency>
 
  <!--junit for unit test-->

diff --git a/src/main/java/com/snowflake/kafka/connector/ConnectorConfigValidator.java b/src/main/java/com/snowflake/kafka/connector/ConnectorConfigValidator.java
@@ -0,0 +1,14 @@
+package com.snowflake.kafka.connector;
+
+import java.util.Map;
+
+public interface ConnectorConfigValidator {
+
+ /**
+ * Validate input configuration
+ *
+ * @param config configuration Map
+ * @return connector name
+ */
+ String validateConfig(Map<String, String> config);
+}