public static PartialRow toRangeBoundToPartialRow(Schema schema, RangePartitionDefinition definition, RangeBoundValue boundValue) { PartialRow partialRow = new PartialRow(schema); if (boundValue != null) { List<Integer> rangeColumns = definition.getColumns().stream() .map(schema::getColumnIndex).collect(toImmutableList()); if (rangeColumns.size() != boundValue.getValues().size()) { throw new IllegalStateException("Expected " + rangeColumns.size() + " range columns, but got " + boundValue.getValues().size()); } for (int i = 0; i < rangeColumns.size(); i++) { Object obj = boundValue.getValues().get(i); int idx = rangeColumns.get(i); ColumnSchema columnSchema = schema.getColumnByIndex(idx); setColumnValue(partialRow, idx, obj, columnSchema.getType(), columnSchema.getName()); } } return partialRow; }
private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(clientSession); Schema schema = table.getSchema(); List<ColumnMetadata> columnsMetaList = schema.getColumns().stream() .filter(column -> !column.isKey() || !column.getName().equals(KuduColumnHandle.ROW_ID)) .map(this::getColumnMetadata) .collect(toImmutableList()); Map<String, Object> properties = clientSession.getTableProperties(tableHandle); return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle; Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int ordinal = 0; ordinal < schema.getColumnCount(); ordinal++) { ColumnSchema col = schema.getColumnByIndex(ordinal); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, ordinal, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
private ColumnMetadata getColumnMetadata(ColumnSchema column) { Map<String, Object> properties = new LinkedHashMap<>(); StringBuilder extra = new StringBuilder(); if (column.isKey()) { properties.put(KuduTableProperties.PRIMARY_KEY, true); extra.append("primary_key, "); } if (column.isNullable()) { properties.put(KuduTableProperties.NULLABLE, true); extra.append("nullable, "); } String encoding = KuduTableProperties.lookupEncodingString(column.getEncoding()); if (!column.getEncoding().equals(ColumnSchema.Encoding.AUTO_ENCODING)) { properties.put(KuduTableProperties.ENCODING, encoding); } extra.append("encoding=").append(encoding).append(", "); String compression = KuduTableProperties.lookupCompressionString(column.getCompressionAlgorithm()); if (!column.getCompressionAlgorithm().equals(ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION)) { properties.put(KuduTableProperties.COMPRESSION, compression); } extra.append("compression=").append(compression); Type prestoType = TypeHelper.fromKuduColumn(column); return new ColumnMetadata(column.getName(), prestoType, null, extra.toString(), false, properties); }
public static PartitionDesign getPartitionDesign(KuduTable table) { Schema schema = table.getSchema(); PartitionDesign partitionDesign = new PartitionDesign(); PartitionSchema partitionSchema = table.getPartitionSchema(); List<HashPartitionDefinition> hashPartitions = partitionSchema.getHashBucketSchemas().stream() .map(hashBucketSchema -> { HashPartitionDefinition hash = new HashPartitionDefinition(); List<String> cols = hashBucketSchema.getColumnIds().stream() .map(idx -> schema.getColumnByIndex(idx).getName()).collect(toImmutableList()); hash.setColumns(cols); hash.setBuckets(hashBucketSchema.getNumBuckets()); return hash; }).collect(toImmutableList()); partitionDesign.setHash(hashPartitions); List<Integer> rangeColumns = partitionSchema.getRangeSchema().getColumns(); if (!rangeColumns.isEmpty()) { RangePartitionDefinition definition = new RangePartitionDefinition(); definition.setColumns(rangeColumns.stream() .map(i -> schema.getColumns().get(i).getName()) .collect(ImmutableList.toImmutableList())); partitionDesign.setRange(definition); } return partitionDesign; }
@Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { List<String> names = Lists.newArrayList(); List<RelDataType> types = Lists.newArrayList(); for (ColumnSchema column : schema.getColumns()) { names.add(column.getName()); RelDataType type = getSqlTypeFromKuduType(typeFactory, column.getType()); type = typeFactory.createTypeWithNullability(type, column.isNullable()); types.add(type); } return typeFactory.createStructType(types, names); }
@Override public Status insert(String table, String key, Map<String, ByteIterator> values) { Insert insert = this.kuduTable.newInsert(); PartialRow row = insert.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { row.addStringUtf8(i, values.get(schema.getColumnByIndex(i).getName()).toArray()); } apply(insert); return Status.OK; }
private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) { Schema schema = table.getSchema(); LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>(); for (ColumnSchema columnSchema : schema.getColumns()) { ColumnDesign design = new ColumnDesign(); design.setNullable(columnSchema.isNullable()); design.setPrimaryKey(columnSchema.isKey()); design.setCompression(lookupCompressionString(columnSchema.getCompressionAlgorithm())); design.setEncoding(lookupEncodingString(columnSchema.getEncoding())); columns.put(columnSchema.getName(), design); } return columns; }
@Override public Status update(String table, String key, Map<String, ByteIterator> values) { Update update = this.kuduTable.newUpdate(); PartialRow row = update.getRow(); row.addString(KEY, key); for (int i = 1; i < schema.getColumnCount(); i++) { String columnName = schema.getColumnByIndex(i).getName(); ByteIterator b = values.get(columnName); if (b != null) { row.addStringUtf8(columnName, b.toArray()); } } apply(update); return Status.OK; }
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
default: throw new IllegalStateException("Unknown type " + schema.getColumnByIndex(i).getType() + " for column " + schema.getColumnByIndex(i).getName());
default: throw new IllegalStateException("Unknown type " + schema.getColumnByIndex(i).getType() + " for column " + schema.getColumnByIndex(i).getName());
ColumnSchema col = schema.getColumnByIndex(i); final String name = col.getName(); final Type kuduType = col.getType(); lastColumnName = name;
if (col.isKey()) rangeKeys.add(col.getName());
private void checkColumn(Type... passedTypes) { for (Type type : passedTypes) { if (this.column.getType().equals(type)) { return; } } throw new IllegalArgumentException(String.format("%s's type isn't %s, it's %s", column.getName(), Arrays.toString(passedTypes), column.getType().getName())); } }
private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(clientSession); Schema schema = table.getSchema(); List<ColumnMetadata> columnsMetaList = schema.getColumns().stream() .filter(column -> !column.isKey() || !column.getName().equals(KuduColumnHandle.ROW_ID)) .map(this::getColumnMetadata) .collect(toImmutableList()); Map<String, Object> properties = clientSession.getTableProperties(tableHandle); return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties); }
@Test public void testGetWrongTypeColumn() { PartialRow partialRow = getPartialRowWithAllTypes(); for (ColumnSchema columnSchema : partialRow.getSchema().getColumns()) { try { callGetByName(partialRow, columnSchema.getName(), getShiftedType(columnSchema.getType())); fail("Expected IllegalArgumentException for type: " + columnSchema.getType()); } catch (IllegalArgumentException ex) { // This is the expected exception. } } }
@Test public void testAddWrongTypeColumn() { PartialRow partialRow = getPartialRowWithAllTypes(); for (ColumnSchema columnSchema : partialRow.getSchema().getColumns()) { try { callAddByName(partialRow, columnSchema.getName(), getShiftedType(columnSchema.getType())); fail("Expected IllegalArgumentException for type: " + columnSchema.getType()); } catch (IllegalArgumentException ex) { // This is the expected exception. } } }