private int getColumnIndex(Schema columns, String colName) { try { return columns.getColumnIndex(colName); } catch (Exception ex) { return -1; } } }
public static Type fromKuduColumn(ColumnSchema column) { return fromKuduClientType(column.getType(), column.getTypeAttributes()); }
for (int i = 0; i < schema.getPrimaryKeyColumnCount(); i++) { switch (schema.getColumnByIndex(i).getType()) { case STRING: to.addStringUtf8(i, from.getString(i).getBytes(StandardCharsets.UTF_8)); break; case INT64: case UNIXTIME_MICROS: to.addLong(i, from.getLong(i)); break; case INT32: to.addInt(i, from.getInt(i)); break; case INT16: to.addShort(i, from.getShort(i)); break; case INT8: to.addByte(i, from.getByte(i)); break; case DOUBLE: to.addDouble(i, from.getDouble(i)); break; case FLOAT: to.addFloat(i, from.getFloat(i)); break; case BOOL: to.addBoolean(i, from.getBoolean(i)); break; case BINARY: to.addBinary(i, from.getBinary(i));
for (int i = 0; i < schema.getPrimaryKeyColumnCount(); i++) { switch (schema.getColumnByIndex(i).getType()) { case STRING: to.addStringUtf8(i, from.getString(i).getBytes(StandardCharsets.UTF_8)); break; case INT64: case UNIXTIME_MICROS: to.addLong(i, from.getLong(i)); break; case INT32: to.addInt(i, from.getInt(i)); break; case INT16: to.addShort(i, from.getShort(i)); break; case INT8: to.addByte(i, from.getByte(i)); break; case DOUBLE: to.addDouble(i, from.getDouble(i)); break; case FLOAT: to.addFloat(i, from.getFloat(i)); break; case BOOL: to.addBoolean(i, from.getBoolean(i)); break; case BINARY: to.addBinary(i, from.getBinary(i));
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
@Override public Status insert(String table, String key, Map<String, ByteIterator> values) { Insert insert = this.kuduTable.newInsert(); PartialRow row = insert.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { row.addStringUtf8(i, values.get(schema.getColumnByIndex(i).getName()).toArray()); } apply(insert); return Status.OK; }
@Override public Status update(String table, String key, Map<String, ByteIterator> values) { Update update = this.kuduTable.newUpdate(); PartialRow row = update.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { String columnName = schema.getColumnByIndex(i).getName(); ByteIterator b = values.get(columnName); if (b != null) { row.addStringUtf8(columnName, b.toArray()); } } apply(update); return Status.OK; }
private static Object toValue(Schema schema, PartialRow bound, Integer idx) { Type type = schema.getColumnByIndex(idx).getType(); switch (type) { case UNIXTIME_MICROS: long millis = bound.getLong(idx) / 1000; return ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC).print(millis); case STRING: return bound.getString(idx); case INT64: return bound.getLong(idx); case INT32: return bound.getInt(idx); case INT16: return bound.getShort(idx); case INT8: return (short) bound.getByte(idx); case BOOL: return bound.getBoolean(idx); case BINARY: return bound.getBinaryCopy(idx); default: throw new IllegalStateException("Unhandled type " + type + " for range partition"); } }
private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) { Schema schema = table.getSchema(); LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>(); for (ColumnSchema columnSchema : schema.getColumns()) { ColumnDesign design = new ColumnDesign(); design.setNullable(columnSchema.isNullable()); design.setPrimaryKey(columnSchema.isKey()); design.setCompression(lookupCompressionString(columnSchema.getCompressionAlgorithm())); design.setEncoding(lookupEncodingString(columnSchema.getEncoding())); columns.put(columnSchema.getName(), design); } return columns; }
private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(clientSession); Schema schema = table.getSchema(); List<ColumnMetadata> columnsMetaList = schema.getColumns().stream() .filter(column -> !column.isKey() || !column.getName().equals(KuduColumnHandle.ROW_ID)) .map(this::getColumnMetadata) .collect(toImmutableList()); Map<String, Object> properties = clientSession.getTableProperties(tableHandle); return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties); }
@Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { List<String> names = Lists.newArrayList(); List<RelDataType> types = Lists.newArrayList(); for (ColumnSchema column : schema.getColumns()) { names.add(column.getName()); RelDataType type = getSqlTypeFromKuduType(typeFactory, column.getType()); type = typeFactory.createTypeWithNullability(type, column.isNullable()); types.add(type); } return typeFactory.createStructType(types, names); }
private ColumnMetadata getColumnMetadata(ColumnSchema column) { Map<String, Object> properties = new LinkedHashMap<>(); StringBuilder extra = new StringBuilder(); if (column.isKey()) { properties.put(KuduTableProperties.PRIMARY_KEY, true); extra.append("primary_key, "); } if (column.isNullable()) { properties.put(KuduTableProperties.NULLABLE, true); extra.append("nullable, "); } String encoding = KuduTableProperties.lookupEncodingString(column.getEncoding()); if (!column.getEncoding().equals(ColumnSchema.Encoding.AUTO_ENCODING)) { properties.put(KuduTableProperties.ENCODING, encoding); } extra.append("encoding=").append(encoding).append(", "); String compression = KuduTableProperties.lookupCompressionString(column.getCompressionAlgorithm()); if (!column.getCompressionAlgorithm().equals(ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION)) { properties.put(KuduTableProperties.COMPRESSION, compression); } extra.append("compression=").append(compression); Type prestoType = TypeHelper.fromKuduColumn(column); return new ColumnMetadata(column.getName(), prestoType, null, extra.toString(), false, properties); }
private ColumnSchema toColumnSchema(ColumnMetadata columnMetadata) { String name = columnMetadata.getName(); ColumnDesign design = KuduTableProperties.getColumnDesign(columnMetadata.getProperties()); Type ktype = TypeHelper.toKuduClientType(columnMetadata.getType()); ColumnSchema.ColumnSchemaBuilder builder = new ColumnSchema.ColumnSchemaBuilder(name, ktype); builder.key(design.isPrimaryKey()).nullable(design.isNullable()); setEncoding(name, builder, design); setCompression(name, builder, design); setTypeAttributes(columnMetadata, builder); return builder.build(); }
public static PartialRow toRangeBoundToPartialRow(Schema schema, RangePartitionDefinition definition, RangeBoundValue boundValue) { PartialRow partialRow = new PartialRow(schema); if (boundValue != null) { List<Integer> rangeColumns = definition.getColumns().stream() .map(schema::getColumnIndex).collect(toImmutableList()); if (rangeColumns.size() != boundValue.getValues().size()) { throw new IllegalStateException("Expected " + rangeColumns.size() + " range columns, but got " + boundValue.getValues().size()); } for (int i = 0; i < rangeColumns.size(); i++) { Object obj = boundValue.getValues().get(i); int idx = rangeColumns.get(i); ColumnSchema columnSchema = schema.getColumnByIndex(idx); setColumnValue(partialRow, idx, obj, columnSchema.getType(), columnSchema.getName()); } } return partialRow; }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle; Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int ordinal = 0; ordinal < schema.getColumnCount(); ordinal++) { ColumnSchema col = schema.getColumnByIndex(ordinal); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, ordinal, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
private void setTypeAttributes(ColumnMetadata columnMetadata, ColumnSchema.ColumnSchemaBuilder builder) { if (columnMetadata.getType() instanceof DecimalType) { DecimalType type = (DecimalType) columnMetadata.getType(); ColumnTypeAttributes attributes = new ColumnTypeAttributes.ColumnTypeAttributesBuilder() .precision(type.getPrecision()) .scale(type.getScale()).build(); builder.typeAttributes(attributes); } }
private Schema buildSchema(List<ColumnMetadata> columns, Map<String, Object> tableProperties) { List<ColumnSchema> kuduColumns = columns.stream() .map(this::toColumnSchema) .collect(ImmutableList.toImmutableList()); return new Schema(kuduColumns); }
private static Type fromKuduClientType(org.apache.kudu.Type ktype, ColumnTypeAttributes attributes) { switch (ktype) { case STRING: return VarcharType.VARCHAR; case UNIXTIME_MICROS: return TimestampType.TIMESTAMP; case INT64: return BigintType.BIGINT; case INT32: return IntegerType.INTEGER; case INT16: return SmallintType.SMALLINT; case INT8: return TinyintType.TINYINT; case FLOAT: return RealType.REAL; case DOUBLE: return DoubleType.DOUBLE; case BOOL: return BooleanType.BOOLEAN; case BINARY: return VarbinaryType.VARBINARY; case DECIMAL: return DecimalType.createDecimalType(attributes.getPrecision(), attributes.getScale()); default: throw new IllegalStateException("Kudu type not implemented for " + ktype); } }
private void setCompression(String name, ColumnSchema.ColumnSchemaBuilder builder, ColumnDesign design) { if (design.getCompression() != null) { try { ColumnSchema.CompressionAlgorithm algorithm = KuduTableProperties.lookupCompression(design.getCompression()); builder.compressionAlgorithm(algorithm); } catch (IllegalArgumentException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unknown compression algorithm " + design.getCompression() + " for column " + name); } } }
private void setEncoding(String name, ColumnSchema.ColumnSchemaBuilder builder, ColumnDesign design) { if (design.getEncoding() != null) { try { ColumnSchema.Encoding encoding = KuduTableProperties.lookupEncoding(design.getEncoding()); builder.encoding(encoding); } catch (IllegalArgumentException e) { throw new PrestoException(GENERIC_INTERNAL_ERROR, "Unknown encoding " + design.getEncoding() + " for column " + name); } } }