List<String> hashPartitionColumns = new ArrayList<>(); hashPartitionColumns.add(KEY); builder.addHashPartitions(hashPartitionColumns, numTablets); builder.setNumReplicas(numReplicas);
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties) { CreateTableOptions options = new CreateTableOptions(); RangePartitionDefinition rangePartitionDefinition = null; PartitionDesign partitionDesign = KuduTableProperties.getPartitionDesign(properties); if (partitionDesign.getHash() != null) { for (HashPartitionDefinition partition : partitionDesign.getHash()) { options.addHashPartitions(partition.getColumns(), partition.getBuckets()); } } if (partitionDesign.getRange() != null) { rangePartitionDefinition = partitionDesign.getRange(); options.setRangePartitionColumns(rangePartitionDefinition.getColumns()); } List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties); if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) { for (RangePartition rangePartition : rangePartitions) { PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower()); PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper()); options.addRangePartition(lower, upper); } } Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties); numReplicas.ifPresent(options::setNumReplicas); return options; }
/** * Add a set of hash partitions to the table. * * Each column must be a part of the table's primary key, and an individual * column may only appear in a single hash component. * * For each set of hash partitions added to the table, the total number of * table partitions is multiplied by the number of buckets. For example, if a * table is created with 3 split rows, and two hash partitions with 4 and 5 * buckets respectively, the total number of table partitions will be 80 * (4 range partitions * 4 hash buckets * 5 hash buckets). * * @param columns the columns to hash * @param buckets the number of buckets to hash into * @return this instance */ public CreateTableOptions addHashPartitions(List<String> columns, int buckets) { addHashPartitions(columns, buckets, 0); return this; }
public static CreateTableOptions getCreateTableOptions(Schema schema, int numReplicas, int rangePartitions, int hashPartitions) { Preconditions.checkArgument(rangePartitions > 0); Preconditions.checkArgument(hashPartitions > 0); CreateTableOptions options = new CreateTableOptions().setNumReplicas(numReplicas); if (rangePartitions > 1) { options.setRangePartitionColumns(ImmutableList.of(COLUMN_KEY_ONE)); BigInteger min = BigInteger.valueOf(Long.MIN_VALUE); BigInteger max = BigInteger.valueOf(Long.MAX_VALUE); BigInteger step = max.multiply(BigInteger.valueOf(2)) .divide(BigInteger.valueOf(rangePartitions)); PartialRow splitRow = schema.newPartialRow(); for (int i = 1; i < rangePartitions; i++) { long key = min.add(step.multiply(BigInteger.valueOf(i))).longValue(); splitRow.addLong(COLUMN_KEY_ONE_IDX, key); options.addSplitRow(splitRow); } } else { options.setRangePartitionColumns(Collections.<String>emptyList()); } if (hashPartitions > 1) { options.addHashPartitions(ImmutableList.of(COLUMN_KEY_ONE), hashPartitions); } return options; }
@Test(timeout = 100000) public void testHashBucketedTable() throws Exception { CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.addHashPartitions(ImmutableList.of("a"), 3); tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42); tableBuilder.setRangePartitionColumns(ImmutableList.<String>of()); testPartitionSchema(tableBuilder); }
public CreateTableOptions getCreateTableOptions() { CreateTableOptions options = new CreateTableOptions(); if(replicas!=null){ options.setNumReplicas(replicas); } if(hasColummns()) { List<String> rangeKeys = new ArrayList<>(); List<String> hashKeys = new ArrayList<>(); for(KuduColumnInfo column : columns){ if(column.isRangeKey()){ rangeKeys.add(column.name()); } if(column.isHashKey()){ hashKeys.add(column.name()); } } options.setRangePartitionColumns(rangeKeys); options.addHashPartitions(hashKeys, replicas*2); } return options; }
@Test(timeout = 100000) public void testHashBucketedAndRangePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.addHashPartitions(ImmutableList.of("a"), 3); tableBuilder.addHashPartitions(ImmutableList.of("b", "c"), 3, 42); tableBuilder.setRangePartitionColumns(ImmutableList.of("c", "b")); PartialRow split = schema.newPartialRow(); split.addString("c", "3"); tableBuilder.addSplitRow(split); split = schema.newPartialRow(); split.addString("c", "3"); split.addString("b", "3"); tableBuilder.addSplitRow(split); testPartitionSchema(tableBuilder); }
if (partitionDesign.getHash() != null) { for (HashPartitionDefinition partition : partitionDesign.getHash()) { options.addHashPartitions(partition.getColumns(), partition.getBuckets());
private void createAndFillSchemasTable(KuduClient client) throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(client); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); try { session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); insert.getRow().addString(0, schemaName); session.apply(insert); } } finally { session.close(); } }
private CreateTableOptions buildCreateTableOptions(Schema schema, Map<String, Object> properties) { CreateTableOptions options = new CreateTableOptions(); RangePartitionDefinition rangePartitionDefinition = null; PartitionDesign partitionDesign = KuduTableProperties.getPartitionDesign(properties); if (partitionDesign.getHash() != null) { for (HashPartitionDefinition partition : partitionDesign.getHash()) { options.addHashPartitions(partition.getColumns(), partition.getBuckets()); } } if (partitionDesign.getRange() != null) { rangePartitionDefinition = partitionDesign.getRange(); options.setRangePartitionColumns(rangePartitionDefinition.getColumns()); } List<RangePartition> rangePartitions = KuduTableProperties.getRangePartitions(properties); if (rangePartitionDefinition != null && !rangePartitions.isEmpty()) { for (RangePartition rangePartition : rangePartitions) { PartialRow lower = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getLower()); PartialRow upper = KuduTableProperties.toRangeBoundToPartialRow(schema, rangePartitionDefinition, rangePartition.getUpper()); options.addRangePartition(lower, upper); } } Optional<Integer> numReplicas = KuduTableProperties.getNumReplicas(properties); numReplicas.ifPresent(options::setNumReplicas); return options; }
tableBuilder.addHashPartitions(ImmutableList.of("a"), 3); tableBuilder.addHashPartitions(ImmutableList.of("b"), 3); tableBuilder.addHashPartitions(ImmutableList.of("c"), 3);
private void createAndFillSchemasTable() throws KuduException { List<String> existingSchemaNames = listSchemaNamesFromTablets(); ColumnSchema tenantColumnSchema = new ColumnSchema.ColumnSchemaBuilder("tenant", Type.STRING) .key(true).build(); ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING) .key(true).build(); Schema schema = new Schema(ImmutableList.of(tenantColumnSchema, schemaColumnSchema)); CreateTableOptions options = new CreateTableOptions(); options.setNumReplicas(1); // TODO config options.addHashPartitions(ImmutableList.of(tenantColumnSchema.getName()), 2); KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options); KuduSession session = client.newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); try { for (String schemaName : existingSchemaNames) { Insert insert = schemasTable.newInsert(); fillSchemaRow(insert.getRow(), schemaName); session.apply(insert); } } finally { session.close(); } }
/** * Creates a new table with two int columns, c0 and c1. c0 is the primary key. * The table is hash partitioned on c0 into two buckets, and range partitioned * with the provided bounds. */ private KuduTable createTable(List<Pair<Integer, Integer>> bounds) throws KuduException { // Create initial table with single range partition covering the entire key // space, and two hash buckets. ArrayList<ColumnSchema> columns = new ArrayList<>(1); columns.add(new ColumnSchema.ColumnSchemaBuilder("c0", Type.INT32) .nullable(false) .key(true) .build()); columns.add(new ColumnSchema.ColumnSchemaBuilder("c1", Type.INT32) .nullable(false) .build()); Schema schema = new Schema(columns); CreateTableOptions createOptions = new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("c0")) .setNumReplicas(1) .addHashPartitions(ImmutableList.of("c0"), 2); for (Pair<Integer, Integer> bound : bounds) { PartialRow lower = schema.newPartialRow(); PartialRow upper = schema.newPartialRow(); lower.addInt("c0", bound.getFirst()); upper.addInt("c0", bound.getSecond()); createOptions.addRangePartition(lower, upper); } return client.createTable(tableName, schema, createOptions); }
builder.addHashPartitions(ImmutableList.of("a"), 2); builder.addHashPartitions(ImmutableList.of("b"), 2); builder.setRangePartitionColumns(ImmutableList.of("a", "b")); List<String> expected = Lists.newArrayList();
builder.addHashPartitions(ImmutableList.of("a"), 2); builder.addHashPartitions(ImmutableList.of("b"), 2); builder.setRangePartitionColumns(ImmutableList.of("a", "b"));
@Before public void setUp() throws Exception { CreateTableOptions builder = new CreateTableOptions(); builder.addHashPartitions( Lists.newArrayList(schema.getColumnByIndex(0).getName()), TABLET_COUNT); table = harness.getClient().createTable(TABLE_NAME, schema, builder); KuduSession session = harness.getClient().newSession(); session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND); // Getting meaty rows. char[] chars = new char[1024]; for (int i = 0; i < ROW_COUNT; i++) { Insert insert = table.newInsert(); PartialRow row = insert.getRow(); row.addInt(0, random.nextInt()); row.addInt(1, i); row.addInt(2, i); row.addString(3, new String(chars)); row.addBoolean(4, true); session.apply(insert); } session.flush(); assertEquals(0, session.countPendingErrors()); }
createOptions.addHashPartitions(ImmutableList.of("key"), 8);
TABLE_NAME, basicSchema, new CreateTableOptions().addHashPartitions(ImmutableList.of("key"), 2)); KuduSession session = client.newSession(); KuduTable table = client.openTable(TABLE_NAME);
@Test(timeout = 100000) public void testHashBucketedAndNonCoveredRangePartitionedTable() throws Exception { Schema schema = createSchema(); CreateTableOptions tableBuilder = new CreateTableOptions(); tableBuilder.setRangePartitionColumns(ImmutableList.of("a", "b", "c")); // Create a non covered range between (3, 5, 6) and (4, 0, 0) PartialRow lowerBoundA = schema.newPartialRow(); lowerBoundA.addString("a", "0"); lowerBoundA.addString("b", "0"); lowerBoundA.addString("c", "0"); PartialRow upperBoundA = schema.newPartialRow(); upperBoundA.addString("a", "3"); upperBoundA.addString("b", "5"); upperBoundA.addString("c", "6"); tableBuilder.addRangePartition(lowerBoundA, upperBoundA); PartialRow lowerBoundB = schema.newPartialRow(); lowerBoundB.addString("a", "4"); lowerBoundB.addString("b", "0"); lowerBoundB.addString("c", "0"); PartialRow upperBoundB = schema.newPartialRow(); upperBoundB.addString("a", "5"); upperBoundB.addString("b", "5"); upperBoundB.addString("c", "6"); tableBuilder.addRangePartition(lowerBoundB, upperBoundB); tableBuilder.addHashPartitions(ImmutableList.of("a", "b", "c"), 4); testPartitionSchema(tableBuilder); }