fix incorrect nested array value

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.4.1
+
+### Bug Fixes
+* incorrect nested array value. [#1221](https:/ClickHouse/clickhouse-java/issues/1221)
+
 ## 0.4.0, 2023-01-19
 ### Breaking changes
 * refactored `JdbcTypeMapping` to make it extensible. [#1075](https:/ClickHouse/clickhouse-java/pull/1075)

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseArraySequence.java

@@ -5,7 +5,7 @@
  */
 public interface ClickHouseArraySequence extends ClickHouseValue {
  /**
- * Allocate an array according to given length. Same as
+ * Allocates an array according to given length. Same as
  * {@code allocate(length, Object.class, 1)}.
  *
  * @param length length of the array
@@ -16,7 +16,7 @@ default ClickHouseArraySequence allocate(int length) {
  }
 
  /**
- * Allocate an array according to given arguments. Same as
+ * Allocates an array according to given arguments. Same as
  * {@code allocate(length, clazz, 1)}.
  *
  * @param length length of the array
@@ -28,7 +28,9 @@ default ClickHouseArraySequence allocate(int length, Class<?> clazz) {
  }
 
  /**
- * Allocate an array according to given arguments.
+ * Allocates an array according to given arguments. Pay attention that this will
+ * will not create new array but reuse existing one, when {@code length} are
+ * {@code clazz} not changed.
  *
  * @param length length of the array
  * @param clazz optional value type, null means {@code Object.class}

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java

@@ -341,10 +341,9 @@ public static Class<?> toWiderPrimitiveType(Class<?> javaClass) {
  javaClass = boolean.class;
  } else if (Byte.class == javaClass || UnsignedByte.class == javaClass || byte.class == javaClass) {
  javaClass = short.class;
- } else if (Integer.class == javaClass || UnsignedInteger.class == javaClass || int.class == javaClass) {
+ } else if (Integer.class == javaClass || UnsignedInteger.class == javaClass || int.class == javaClass
+ || Long.class == javaClass || UnsignedLong.class == javaClass) {
  javaClass = long.class;
- } else if (Long.class == javaClass || long.class == javaClass) {
- javaClass = UnsignedLong.class;
  } else if (Short.class == javaClass || UnsignedShort.class == javaClass || short.class == javaClass
  || Character.class == javaClass || char.class == javaClass) {
  javaClass = int.class;

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDeserializer.java

@@ -12,6 +12,19 @@
  */
 @FunctionalInterface
 public interface ClickHouseDeserializer {
+ static final class ResetValueDeserializer implements ClickHouseDeserializer {
+ private final ClickHouseDeserializer deserializer;
+
+ public ResetValueDeserializer(ClickHouseDeserializer deserializer) {
+ this.deserializer = deserializer;
+ }
+
+ @Override
+ public ClickHouseValue deserialize(ClickHouseValue ref, ClickHouseInputStream input) throws IOException {
+ return deserializer.deserialize(ref.resetToDefault(), input);
+ }
+ }
+
  static class CompositeDeserializer implements ClickHouseDeserializer {
  protected final ClickHouseDeserializer[] deserializers;
 

clickhouse-data/src/main/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessor.java

@@ -311,23 +311,25 @@ public ClickHouseRowBinaryProcessor(ClickHouseDataConfig config, ClickHouseInput
 
  protected ClickHouseDeserializer[] getArrayDeserializers(ClickHouseDataConfig config,
  List<ClickHouseColumn> columns) {
- List<ClickHouseDeserializer> list = new ArrayList<>(columns.size());
+ ClickHouseDeserializer[] array = new ClickHouseDeserializer[columns.size()];
  ClickHouseDataConfig modifiedConfig = new UseObjectConfig(config);
+ int i = 0;
  for (ClickHouseColumn column : columns) {
- list.add(getDeserializer(modifiedConfig,
- ClickHouseColumn.of(column.getColumnName(), ClickHouseDataType.Array, false, column)));
+ array[i++] = getDeserializer(modifiedConfig,
+ ClickHouseColumn.of(column.getColumnName(), ClickHouseDataType.Array, false, column));
  }
- return list.toArray(new ClickHouseDeserializer[0]);
+ return array;
  }
 
  protected ClickHouseSerializer[] getArraySerializers(ClickHouseDataConfig config, List<ClickHouseColumn> columns) {
- List<ClickHouseSerializer> list = new ArrayList<>(columns.size());
+ ClickHouseSerializer[] array = new ClickHouseSerializer[columns.size()];
  ClickHouseDataConfig modifiedConfig = new UseObjectConfig(config);
+ int i = 0;
  for (ClickHouseColumn column : columns) {
- list.add(getSerializer(modifiedConfig,
- ClickHouseColumn.of(column.getColumnName(), ClickHouseDataType.Array, false, column)));
+ array[i++] = getSerializer(modifiedConfig,
+ ClickHouseColumn.of(column.getColumnName(), ClickHouseDataType.Array, false, column));
  }
- return list.toArray(new ClickHouseSerializer[0]);
+ return array;
  }
 
  @Override
@@ -462,29 +464,35 @@ public ClickHouseDeserializer getDeserializer(ClickHouseDataConfig config, Click
  break;
  // nested
  case Array: {
- ClickHouseColumn baseColumn = column.getArrayBaseColumn();
- Class<?> javaClass = baseColumn.getObjectClassForArray(config);
- if (column.getArrayNestedLevel() == 1 && !baseColumn.isNullable() && javaClass.isPrimitive()) {
- int byteLength = baseColumn.getDataType().getByteLength();
- if (byteLength == Byte.BYTES) { // Bool, *Int8
- deserializer = BinaryDataProcessor::readByteArray;
- } else if (byteLength == Short.BYTES) { // *Int16
- deserializer = BinaryDataProcessor::readShortArray;
- } else if (int.class == javaClass) { // Int32
- deserializer = BinaryDataProcessor::readIntegerArray;
- } else if (long.class == javaClass) { // UInt32, *Int64
- deserializer = byteLength == Long.BYTES ? BinaryDataProcessor::readLongArray
- : BinaryDataProcessor::readIntegerArray;
- } else if (float.class == javaClass) { // Float32
- deserializer = BinaryDataProcessor::readFloatArray;
- } else if (double.class == javaClass) { // Float64
- deserializer = BinaryDataProcessor::readDoubleArray;
+ if (column.getArrayNestedLevel() == 1) {
+ ClickHouseColumn baseColumn = column.getArrayBaseColumn();
+ Class<?> javaClass = baseColumn.getObjectClassForArray(config);
+ if (!baseColumn.isNullable() && javaClass.isPrimitive()) {
+ int byteLength = baseColumn.getDataType().getByteLength();
+ if (byteLength == Byte.BYTES) { // Bool, *Int8
+ deserializer = BinaryDataProcessor::readByteArray;
+ } else if (byteLength == Short.BYTES) { // *Int16
+ deserializer = BinaryDataProcessor::readShortArray;
+ } else if (int.class == javaClass) { // Int32
+ deserializer = BinaryDataProcessor::readIntegerArray;
+ } else if (long.class == javaClass) { // UInt32, *Int64
+ deserializer = byteLength == Long.BYTES ? BinaryDataProcessor::readLongArray
+ : BinaryDataProcessor::readIntegerArray;
+ } else if (float.class == javaClass) { // Float32
+ deserializer = BinaryDataProcessor::readFloatArray;
+ } else if (double.class == javaClass) { // Float64
+ deserializer = BinaryDataProcessor::readDoubleArray;
+ } else {
+ throw new IllegalArgumentException("Unsupported primitive type: " + javaClass);
+ }
  } else {
- throw new IllegalArgumentException("Unsupported primitive type: " + javaClass);
+ deserializer = new BinaryDataProcessor.ArrayDeserializer(config, column, true,
+ getDeserializer(config, column.getNestedColumns().get(0)));
  }
  } else {
  deserializer = new BinaryDataProcessor.ArrayDeserializer(config, column, true,
- getDeserializer(config, column.getNestedColumns().get(0)));
+ new ClickHouseDeserializer.ResetValueDeserializer(
+ getDeserializer(config, column.getNestedColumns().get(0))));
  }
  break;
  }

clickhouse-data/src/main/java/com/clickhouse/data/value/array/ClickHouseLongArrayValue.java

@@ -77,7 +77,7 @@ public UnsignedLongArrayValue update(String value) {
  long[] arr = new long[list.size()];
  int index = 0;
  for (String v : list) {
- arr[index++] = v == null ? 0 : Integer.parseUnsignedInt(v);
+ arr[index++] = v == null ? 0L : Long.parseUnsignedLong(v);
  }
  set(arr);
  }

clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java

@@ -9,6 +9,11 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.clickhouse.data.value.ClickHouseArrayValue;
+import com.clickhouse.data.value.ClickHouseLongValue;
+import com.clickhouse.data.value.UnsignedLong;
+import com.clickhouse.data.value.array.ClickHouseLongArrayValue;
+
 public class ClickHouseColumnTest {
  @DataProvider(name = "enumTypesProvider")
  private Object[][] getEnumTypes() {
@@ -257,6 +262,16 @@ public boolean isWidenUnsignedTypes() {
  Assert.assertEquals(v.update(new Long[] { 1L }).asObject(), new Long[] { 1L });
  v = ClickHouseColumn.of("a", "Array(Array(UInt16))").newValue(config);
  Assert.assertEquals(v.asObject(), new int[0][]);
+ v = ClickHouseColumn.of("a", "Array(UInt64)").newValue(config);
+ Assert.assertEquals(v.asObject(), new UnsignedLong[0]);
+ Assert.assertEquals(((ClickHouseLongArrayValue) v).allocate(1)
+ .setValue(0, ClickHouseLongValue.of(1L)).asObject(), new long[] { 1L });
+ Assert.assertEquals(((ClickHouseLongArrayValue) v).allocate(1)
+ .setValue(0, ClickHouseLongValue.ofUnsigned(1L)).asObject(), new long[] { 1L });
+ v = ClickHouseColumn.of("a", "Array(Array(UInt64))").newValue(config);
+ Assert.assertEquals(v.asObject(), new long[0][]);
+ Assert.assertEquals(((ClickHouseArrayValue<?>) v).allocate(1)
+ .setValue(0, ClickHouseLongArrayValue.of(new long[] { 1L })).asObject(), new long[][] { { 1L } });
  v = ClickHouseColumn.of("a", "Array(Array(Array(UInt8)))").newValue(config);
  Assert.assertEquals(v.asObject(), new short[0][][]);
  v = ClickHouseColumn.of("a", "Array(Array(Array(Nullable(UInt8))))").newValue(config);

clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseRowBinaryProcessorTest.java

@@ -156,6 +156,34 @@ protected byte[] getRawData(String typeName, String key) {
  data = toBytes(4, 0, 1, 66, 2, 66, 67, 3, 66, 67, 68);
  }
  break;
+ case "Array(Array(Array(UInt8)))":
+ if ("[[3]],[[1,2],[2,1]],[[4,5],[5,4]]".equals(key)) {
+ data = toBytes(3, 1, 1, 3, 2, 2, 1, 2, 2, 2, 1, 2, 2, 4, 5, 2, 5, 4);
+ }
+ break;
+ case "Array(Array(UInt64))":
+ if ("[1,2,3],[3,2,1],[4,5]".equals(key)) {
+ data = toBytes(3, 3, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 3,
+ 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0,
+ 0, 5, 0, 0, 0, 0, 0, 0, 0);
+ }
+ break;
+ case "Map(String,Array(UInt8))":
+ if ("[1,2,3],[3,2,1],[4,5]".equals(key)) {
+ data = toBytes(3, 1, 0x61, 3, 1, 2, 3, 1, 0x62, 3, 3, 2, 1, 1, 0x63, 2, 4, 5);
+ }
+ break;
+ case "Tuple(Array(UInt8),Array(UInt8),Array(UInt8))":
+ if ("[1,2,3],[3,2,1],[4,5]".equals(key)) {
+ data = toBytes(3, 1, 2, 3, 3, 3, 2, 1, 2, 4, 5);
+ }
+ break;
+ case "Array(Array(String))":
+ if ("[foo,bar],[qaz,qux]".equals(key)) {
+ data = toBytes(2, 2, 3, 0X66, 0X6F, 0X6F, 3, 0X62, 0X61, 0X72, 2, 3, 0X71, 0X61, 0X7A, 3, 0X71,
+ 0X75, 0X78);
+ }
+ break;
  case "Bool":
  case "Int8":
  case "UInt8":

clickhouse-data/src/test/java/com/clickhouse/data/format/ClickHouseTabSeparatedProcessorTest.java

@@ -150,8 +150,7 @@ protected byte[] getRawData(String typeName, String key) {
 
  @Override
  public void testDeserializeNestedTypes(ClickHouseDataConfig config, String typeName, String dataKey,
- String valueClass,
- Object arrVal, Object objVal, String strVal, String sqlExpr) throws IOException {
+ String valueClass, Object arrVal, Object objVal, String strVal, String sqlExpr) throws IOException {
  throw new SkipException("Skip as it's not fully implemented");
  }