@Override public void updateSchema(VectorAccessible batch) throws IOException { BatchSchema schema = batch.getSchema(); int i = 0; try { if (!checkForTable(name)) { List<ColumnSchema> columns = new ArrayList<>(); for (MaterializedField f : schema) { columns.add(new ColumnSchema.ColumnSchemaBuilder(f.getName(), getType(f.getType())) .nullable(f.getType().getMode() == DataMode.OPTIONAL) .key(i == 0).build()); i++; } Schema kuduSchema = new Schema(columns); table = client.createTable(name, kuduSchema, new CreateTableOptions()); } } catch (Exception e) { throw new IOException(e); } }
columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
public static Schema createSchemaWithBinaryColumns() { ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(); columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.BINARY).key(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.DOUBLE).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.BINARY).nullable(true).build()); return new Schema(columns); }
/** * Creates a new table with two int columns, c0 and c1. c0 is the primary key. * The table is hash partitioned on c0 into two buckets, and range partitioned * with the provided bounds. */ private KuduTable createTable(List<Pair<Integer, Integer>> bounds) throws KuduException { // Create initial table with single range partition covering the entire key // space, and two hash buckets. ArrayList<ColumnSchema> columns = new ArrayList<>(1); columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.INT32) .nullable(false) .key(true) .build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.INT32) .nullable(false) .build()); Schema schema = new Schema(columns); CreateTableOptions createOptions = new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("c0")) .setNumReplicas(1) .addHashPartitions(ImmutableList.of("c0"), 2); for (Pair<Integer, Integer> bound : bounds) { PartialRow lower = schema.newPartialRow(); PartialRow upper = schema.newPartialRow(); lower.addInt("c0", bound.getFirst()); upper.addInt("c0", bound.getSecond()); createOptions.addRangePartition(lower, upper); } return client.createTable(tableName, schema, createOptions); }
public static ColumnSchema pbToColumnSchema(Common.ColumnSchemaPB pb) { Type type = Type.getTypeForDataType(pb.getType()); ColumnTypeAttributes typeAttributes = pb.hasTypeAttributes() ? pbToColumnTypeAttributes(pb.getTypeAttributes()) : null; Object defaultValue = pb.hasWriteDefaultValue() ? byteStringToObject(type, typeAttributes, pb.getWriteDefaultValue()) : null; ColumnSchema.Encoding encoding = ColumnSchema.Encoding.valueOf(pb.getEncoding().name()); ColumnSchema.CompressionAlgorithm compressionAlgorithm = ColumnSchema.CompressionAlgorithm.valueOf(pb.getCompression().name()); int desiredBlockSize = pb.getCfileBlockSize(); return new ColumnSchema.ColumnSchemaBuilder(pb.getName(), type) .key(pb.getIsKey()) .nullable(pb.getIsNullable()) .defaultValue(defaultValue) .encoding(encoding) .compressionAlgorithm(compressionAlgorithm) .desiredBlockSize(desiredBlockSize) .typeAttributes(typeAttributes) .build(); }
public static Schema getSchemaWithAllTypes() { List<ColumnSchema> columns = ImmutableList.of( new ColumnSchema.ColumnSchemaBuilder("int8", Type.INT8).key(true).build(), new ColumnSchema.ColumnSchemaBuilder("int16", Type.INT16).build(), new ColumnSchema.ColumnSchemaBuilder("int32", Type.INT32).build(), new ColumnSchema.ColumnSchemaBuilder("int64", Type.INT64).build(), new ColumnSchema.ColumnSchemaBuilder("bool", Type.BOOL).build(), new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).build(), new ColumnSchema.ColumnSchemaBuilder("double", Type.DOUBLE).build(), new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).build(), new ColumnSchema.ColumnSchemaBuilder("binary-array", Type.BINARY).build(), new ColumnSchema.ColumnSchemaBuilder("binary-bytebuffer", Type.BINARY).build(), new ColumnSchema.ColumnSchemaBuilder("null", Type.STRING).nullable(true).build(), new ColumnSchema.ColumnSchemaBuilder("timestamp", Type.UNIXTIME_MICROS).build(), new ColumnSchema.ColumnSchemaBuilder("decimal", Type.DECIMAL) .typeAttributes(DecimalUtil.typeAttributes(5, 3)).build()); return new Schema(columns); }
public static Schema createSchemaWithDecimalColumns() { ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(); columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.DECIMAL).key(true) .typeAttributes( new ColumnTypeAttributes.ColumnTypeAttributesBuilder() .precision(DecimalUtil.MAX_DECIMAL64_PRECISION).build() ).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.DECIMAL).nullable(true) .typeAttributes( new ColumnTypeAttributes.ColumnTypeAttributesBuilder() .precision(DecimalUtil.MAX_DECIMAL128_PRECISION).build() ).build()); return new Schema(columns); } }
private static Schema getSchema() { ArrayList<ColumnSchema> columns = new ArrayList<>(3); columns.add(new ColumnSchema.ColumnSchemaBuilder("key1", STRING) .key(true) .build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("key2", STRING) .key(true) .build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("val", STRING) .nullable(true) // Important because we need to make sure it gets passed in projections .build()); return new Schema(columns); } }
public static Schema getBasicSchema() { ArrayList<ColumnSchema> columns = new ArrayList<>(5); columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("column1_i", Type.INT32).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("column2_i", Type.INT32).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("column3_s", Type.STRING) .nullable(true) .desiredBlockSize(4096) .encoding(ColumnSchema.Encoding.DICT_ENCODING) .compressionAlgorithm(ColumnSchema.CompressionAlgorithm.LZ4) .build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("column4_b", Type.BOOL).build()); return new Schema(columns); }
/** * Add a new column that's nullable. * @param name name of the new column * @param type type of the new column * @param defaultVal the default value of the new column * @return this instance */ public AlterTableOptions addNullableColumn(String name, Type type, Object defaultVal) { return addColumn(new ColumnSchema.ColumnSchemaBuilder(name, type) .nullable(true) .defaultValue(defaultVal) .build()); }
private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata) { String name = columnMetadata.getName(); ColumnDesign design = KuduTableProperties.getColumnDesign(columnMetadata.getProperties()); Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType()); ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype); builder.key(design.isPrimaryKey()).nullable(design.isNullable()); setEncoding(name, builder, design); setCompression(name, builder, design); setTypeAttributes(columnMetadata, builder); return builder.build(); }
protected ColumnSchema columnSchema() { return new ColumnSchema.ColumnSchemaBuilder(name, type) .key(key) .nullable(nullable) .defaultValue(defaultValue) .desiredBlockSize(blockSize) .encoding(encoding.encode) .compressionAlgorithm(compression.algorithm) .build(); }
/** * Clone the given column schema instance. The new instance will include only the name, type, and * nullability of the passed one. * @return a new column schema */ private static ColumnSchema getStrippedColumnSchema(ColumnSchema columnToClone) { return new ColumnSchema.ColumnSchemaBuilder(columnToClone.getName(), columnToClone.getType()) .nullable(columnToClone.isNullable()) .typeAttributes(columnToClone.getTypeAttributes()) .build(); }
private Schema createManyStringsSchema() { ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4); columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.STRING).key(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build()); return new Schema(columns); }
private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata, Map<String, ColumnDesign> columnDesignMap) { String name = columnMetadata.getName(); ColumnDesign design = columnDesignMap.getOrDefault(name, ColumnDesign.DEFAULT); Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType()); ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype); builder.key(design.isKey()).nullable(design.isNullable()); setEncoding(name, builder, design); setCompression(name, builder, design); setTypeAttributes(columnMetadata, builder); return builder.build(); }
public static Schema getTableSchema() { return new Schema(ImmutableList.of( new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_ONE, Type.INT64).key(true).build(), new ColumnSchema.ColumnSchemaBuilder(COLUMN_KEY_TWO, Type.INT64).key(true).build(), new ColumnSchema.ColumnSchemaBuilder(COLUMN_PREV_ONE, Type.INT64).nullable(true).build(), new ColumnSchema.ColumnSchemaBuilder(COLUMN_PREV_TWO, Type.INT64).nullable(true).build(), new ColumnSchema.ColumnSchemaBuilder(COLUMN_ROW_ID, Type.INT64).build(), new ColumnSchema.ColumnSchemaBuilder(COLUMN_CLIENT, Type.STRING).build(), new ColumnSchema.ColumnSchemaBuilder(COLUMN_UPDATE_COUNT, Type.INT32).build() )); }
private Schema createTableSchema(Type type) { ColumnSchema key = new ColumnSchema.ColumnSchemaBuilder("key", Type.INT64).key(true).build(); ColumnSchema val = new ColumnSchema.ColumnSchemaBuilder("value", type).nullable(true).build(); return new Schema(ImmutableList.of(key, val)); }
public static Schema createSchemaWithTimestampColumns() { ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(); columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.UNIXTIME_MICROS).key(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.UNIXTIME_MICROS).nullable(true).build()); return new Schema(columns); }
public static Schema createManyStringsSchema() { ArrayList<ColumnSchema> columns = new ArrayList<ColumnSchema>(4); columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.STRING).key(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.STRING).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c2", Type.STRING).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c3", Type.STRING).nullable(true).build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c4", Type.STRING).nullable(true).build()); return new Schema(columns); }
private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata) { String name = columnMetadata.getName(); ColumnDesign design = KuduTableProperties.getColumnDesign(columnMetadata.getProperties()); Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType()); ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype); builder.key(design.isPrimaryKey()).nullable(design.isNullable()); setEncoding(name, builder, design); setCompression(name, builder, design); setTypeAttributes(columnMetadata, builder); return builder.build(); }