@VisibleForTesting Schema newSchema() { return Schema.builder().add("record", Types.JSON).build(); // generate a schema }
private Schema applyTruncateRule(Schema inputSchema, TruncateRule rule) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { if (column.getName().length() <= rule.getMaxLength()) { builder.add(column.getName(), column.getType()); } else { try { builder.add(column.getName().substring(0, rule.getMaxLength()), column.getType()); } catch (IndexOutOfBoundsException ex) { logger.error("FATAL unexpected error in \"truncate\" rule: substring failed."); throw new AssertionError("FATAL unexpected error in \"truncate\" rule: substring failed.", ex); } } } return builder.build(); }
private Schema applyLowerToUpperRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toUpperCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
@Test public void testRenewPageWithStrings() { this.bufferAllocator = new BufferAllocator() { @Override public Buffer allocate() { return Buffer.allocate(1); } @Override public Buffer allocate(int minimumCapacity) { return Buffer.allocate(minimumCapacity); } }; assertEquals( 3, buildPages( Schema.builder() .add("col1", LONG) .add("col1", STRING) .build(), 0L, "record0", 1L, "record1", 3L, "record3" ).size()); }
@Test public void testRenewPage() { this.bufferAllocator = new BufferAllocator() { @Override public Buffer allocate() { return Buffer.allocate(1); } @Override public Buffer allocate(int minimumCapacity) { return Buffer.allocate(minimumCapacity); } }; assertEquals( 9, buildPages(Schema.builder().add("col1", LONG).build(), 0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L).size()); }
private Schema applyRegexReplaceRule(Schema inputSchema, RegexReplaceRule rule) { final String match = rule.getMatch(); final String replace = rule.getReplace(); Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { // TODO(dmikurube): Check if we need a kind of sanitization? try { builder.add(column.getName().replaceAll(match, replace), column.getType()); } catch (PatternSyntaxException ex) { throw new ConfigException(ex); } } return builder.build(); }
@Test public void testJson() { check(Schema.builder().add("col1", JSON).build(), getJsonSampleData()); }
@Test public void testMixedTypes() { check(Schema.builder() .add("col3", DOUBLE) .add("col1", STRING) .add("col3", LONG) .add("col3", BOOLEAN) .add("col2", TIMESTAMP) .add("col4", JSON) .build(), 8122.0, "val1", 3L, false, Timestamp.ofEpochMilli(0), getJsonSampleData(), 140.15, "val2", Long.MAX_VALUE, true, Timestamp.ofEpochMilli(10), getJsonSampleData()); }
private Schema makeSchema(final String[] columnNames) { Schema.Builder builder = new Schema.Builder(); for (String columnName : columnNames) { builder.add(columnName, STRING); } return builder.build(); }
private Schema applyUpperToLowerRule(Schema inputSchema) { Schema.Builder builder = Schema.builder(); for (Column column : inputSchema.getColumns()) { builder.add(column.getName().toLowerCase(Locale.ENGLISH), column.getType()); } return builder.build(); }
@Test public void testTimestamp() { check(Schema.builder().add("col1", TIMESTAMP).build(), Timestamp.ofEpochMilli(0), Timestamp.ofEpochMilli(10)); }
@Test public void testUniqueStrings() { check(Schema.builder().add("col1", STRING).build(), "test1", "test2", "test0"); }
@Test public void testDouble() { check(Schema.builder().add("col1", DOUBLE).build(), 8.1, 3.141592, 4.3); }
@Test public void testDuplicateStrings() { check(Schema.builder().add("col1", STRING).build(), "test1", "test1", "test1"); }
@Test public void testRepeatableFlush() { MockPageOutput output = new MockPageOutput(); this.builder = new PageBuilder(bufferAllocator, Schema.builder().add("col1", STRING).build(), output); builder.flush(); builder.flush(); } }
@Test public void testRepeatableClose() { MockPageOutput output = new MockPageOutput(); this.builder = new PageBuilder(bufferAllocator, Schema.builder().add("col1", STRING).build(), output); builder.close(); builder.close(); }
@Test public void testLong() { check(Schema.builder().add("col1", LONG).build(), 1L, Long.MIN_VALUE, Long.MAX_VALUE); }
@Test public void testBoolean() { check(Schema.builder().add("col1", BOOLEAN).build(), false, true, true); }