private ColumnMetadata getColumnMetadata(ColumnSchema column) { Map<String, Object> properties = new LinkedHashMap<>(); StringBuilder extra = new StringBuilder(); if (column.isKey()) { properties.put(KuduTableProperties.PRIMARY_KEY, true); extra.append("primary_key, "); } if (column.isNullable()) { properties.put(KuduTableProperties.NULLABLE, true); extra.append("nullable, "); } String encoding = KuduTableProperties.lookupEncodingString(column.getEncoding()); if (!column.getEncoding().equals(ColumnSchema.Encoding.AUTO_ENCODING)) { properties.put(KuduTableProperties.ENCODING, encoding); } extra.append("encoding=").append(encoding).append(", "); String compression = KuduTableProperties.lookupCompressionString(column.getCompressionAlgorithm()); if (!column.getCompressionAlgorithm().equals(ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION)) { properties.put(KuduTableProperties.COMPRESSION, compression); } extra.append("compression=").append(compression); Type prestoType = TypeHelper.fromKuduColumn(column); return new ColumnMetadata(column.getName(), prestoType, null, extra.toString(), false, properties); }
private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) { Schema schema = table.getSchema(); LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>(); for (ColumnSchema columnSchema : schema.getColumns()) { ColumnDesign design = new ColumnDesign(); design.setNullable(columnSchema.isNullable()); design.setPrimaryKey(columnSchema.isKey()); design.setCompression(lookupCompressionString(columnSchema.getCompressionAlgorithm())); design.setEncoding(lookupEncodingString(columnSchema.getEncoding())); columns.put(columnSchema.getName(), design); } return columns; }
private ColumnMetadata getColumnMetadata(ColumnSchema column) { Map<String, Object> properties = new LinkedHashMap<>(); StringBuilder extra = new StringBuilder(); if (column.isKey()) { properties.put(KuduTableProperties.PRIMARY_KEY, true); extra.append("primary_key, "); } if (column.isNullable()) { properties.put(KuduTableProperties.NULLABLE, true); extra.append("nullable, "); } String encoding = KuduTableProperties.lookupEncodingString(column.getEncoding()); if (!column.getEncoding().equals(ColumnSchema.Encoding.AUTO_ENCODING)) { properties.put(KuduTableProperties.ENCODING, encoding); } extra.append("encoding=").append(encoding).append(", "); String compression = KuduTableProperties.lookupCompressionString(column.getCompressionAlgorithm()); if (!column.getCompressionAlgorithm().equals(ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION)) { properties.put(KuduTableProperties.COMPRESSION, compression); } extra.append("compression=").append(compression); Type prestoType = TypeHelper.fromKuduColumn(column); return new ColumnMetadata(column.getName(), prestoType, null, extra.toString(), false, properties); }
private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(clientSession); Schema schema = table.getSchema(); List<ColumnMetadata> columnsMetaList = schema.getColumns().stream() .filter(col -> !col.isKey() || !col.getName().equals(KuduColumnHandle.ROW_ID)) .map(col -> { StringBuilder extra = new StringBuilder(); if (col.isKey()) { extra.append("key, "); } else if (col.isNullable()) { extra.append("nullable, "); } if (col.getEncoding() != null) { extra.append("encoding=").append(col.getEncoding().name()).append(", "); } if (col.getCompressionAlgorithm() != null) { extra.append("compression=").append(col.getCompressionAlgorithm().name()).append(", "); } if (extra.length() > 2) { extra.setLength(extra.length() - 2); } Type prestoType = TypeHelper.fromKuduColumn(col); return new ColumnMetadata(col.getName(), prestoType, null, extra.toString(), false); }).collect(toImmutableList()); Map<String, Object> properties = clientSession.getTableProperties(tableHandle); return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties); }
private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) { Schema schema = table.getSchema(); LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>(); for (ColumnSchema columnSchema : schema.getColumns()) { ColumnDesign design = new ColumnDesign(); design.setNullable(columnSchema.isNullable()); design.setKey(columnSchema.isKey()); if (columnSchema.getCompressionAlgorithm() != null) { design.setCompression(columnSchema.getCompressionAlgorithm().name()); } if (columnSchema.getEncoding() != null) { design.setEncoding(columnSchema.getEncoding().name()); } columns.put(columnSchema.getName(), design); } return columns; }
private static LinkedHashMap<String, ColumnDesign> getColumns(KuduTable table) { Schema schema = table.getSchema(); LinkedHashMap<String, ColumnDesign> columns = new LinkedHashMap<>(); for (ColumnSchema columnSchema : schema.getColumns()) { ColumnDesign design = new ColumnDesign(); design.setNullable(columnSchema.isNullable()); design.setPrimaryKey(columnSchema.isKey()); design.setCompression(lookupCompressionString(columnSchema.getCompressionAlgorithm())); design.setEncoding(lookupEncodingString(columnSchema.getEncoding())); columns.put(columnSchema.getName(), design); } return columns; }
public static Common.ColumnSchemaPB columnToPb(Common.ColumnSchemaPB.Builder schemaBuilder, ColumnSchema column) { schemaBuilder .setName(column.getName()) .setType(column.getType().getDataType(column.getTypeAttributes())) .setIsKey(column.isKey()) .setIsNullable(column.isNullable()) .setCfileBlockSize(column.getDesiredBlockSize()); if (column.getEncoding() != null) { schemaBuilder.setEncoding(column.getEncoding().getInternalPbType()); } if (column.getCompressionAlgorithm() != null) { schemaBuilder.setCompression(column.getCompressionAlgorithm().getInternalPbType()); } if (column.getDefaultValue() != null) { schemaBuilder.setReadDefaultValue(UnsafeByteOperations.unsafeWrap( objectToWireFormat(column, column.getDefaultValue()))); } if(column.getTypeAttributes() != null) { schemaBuilder.setTypeAttributes( columnTypeAttributesToPb(Common.ColumnTypeAttributesPB.newBuilder(), column)); } return schemaBuilder.build(); }
@Test public void testAlterModifyColumns() throws Exception { KuduTable table = createTable(ImmutableList.<Pair<Integer,Integer>>of()); insertRows(table, 0, 100); assertEquals(100, countRowsInTable(table)); // Check for expected defaults. ColumnSchema col = table.getSchema().getColumns().get(1); assertEquals(CompressionAlgorithm.DEFAULT_COMPRESSION, col.getCompressionAlgorithm()); assertEquals(Encoding.AUTO_ENCODING, col.getEncoding()); assertEquals(null, col.getDefaultValue()); // Alter the table. client.alterTable(tableName, new AlterTableOptions() .changeCompressionAlgorithm(col.getName(), CompressionAlgorithm.SNAPPY) .changeEncoding(col.getName(), Encoding.RLE) .changeDefault(col.getName(), 0)); // Check for new values. table = client.openTable(tableName); col = table.getSchema().getColumns().get(1); assertEquals(CompressionAlgorithm.SNAPPY, col.getCompressionAlgorithm()); assertEquals(Encoding.RLE, col.getEncoding()); assertEquals(0, col.getDefaultValue()); }
/** * Test creating and deleting a table through a KuduClient. */ @Test(timeout = 100000) public void testCreateDeleteTable() throws Exception { // Check that we can create a table. client.createTable(TABLE_NAME, basicSchema, getBasicCreateTableOptions()); assertFalse(client.getTablesList().getTablesList().isEmpty()); assertTrue(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can delete it. client.deleteTable(TABLE_NAME); assertFalse(client.getTablesList().getTablesList().contains(TABLE_NAME)); // Check that we can re-recreate it, with a different schema. List<ColumnSchema> columns = new ArrayList<>(basicSchema.getColumns()); columns.add(new ColumnSchema.ColumnSchemaBuilder("one more", Type.STRING).build()); Schema newSchema = new Schema(columns); client.createTable(TABLE_NAME, newSchema, getBasicCreateTableOptions()); // Check that we can open a table and see that it has the new schema. KuduTable table = client.openTable(TABLE_NAME); assertEquals(newSchema.getColumnCount(), table.getSchema().getColumnCount()); assertTrue(table.getPartitionSchema().isSimpleRangePartitioning()); // Check that the block size parameter we specified in the schema is respected. assertEquals(4096, newSchema.getColumn("column3_s").getDesiredBlockSize()); assertEquals(ColumnSchema.Encoding.DICT_ENCODING, newSchema.getColumn("column3_s").getEncoding()); assertEquals(ColumnSchema.CompressionAlgorithm.LZ4, newSchema.getColumn("column3_s").getCompressionAlgorithm()); }
assertEquals("wrong compression algorithm", ColumnSchema.CompressionAlgorithm.NO_COMPRESSION, table.getSchema().getColumn("value").getCompressionAlgorithm()); assertEquals("wrong compression algorithm post alter", ColumnSchema.CompressionAlgorithm.SNAPPY, reopenedTable.getSchema().getColumn("value").getCompressionAlgorithm());