private void checkSqlColumnsMatch(String sqlTable) { Set<String> expectedColumns = Sets.newHashSet(); for (ColumnSchema column : schema.getColumns()) { expectedColumns.add(column.getColumnName()); } List<String> invalidColumns = Lists.newLinkedList(); for (String column : columns) { if (!expectedColumns.contains(column)) { LOG.error("Found non-existent column " + column); invalidColumns.add(column); } } if (invalidColumns.size() > 0) { String expectedColumnString = Joiner.on(",").join(expectedColumns); String invalidColumnString = Joiner.on(",").join(invalidColumns); String message = String.format("In table %s following columns (%s)" + " are not valid columns. Expected columns (%s)", sqlTable, invalidColumnString, expectedColumnString); throw new IllegalStateException(message); } }
/** * Create a {@link ColumnSchema} based on fields in the builder. * * @return Column Schema */ public ColumnSchema build() { return new ColumnSchema(columnName, type, isNullable, isAutoIncrement, maxLength, scale, precision); } }
private static byte[] encodeValue(final ByteBuffer value, final ColumnSchema columnSchema) { try { switch (columnSchema.getType()) { case LONG: case TIME: { final long longValue = value.getLong(); return Bytes.toBytes(longValue ^ INVERT_SIGN_MASK); } case DOUBLE: { final double doubleValue = value.getDouble(); final long longValue = Double.doubleToLongBits(doubleValue); if (doubleValue < 0.0) { return Bytes.toBytes(~longValue); } return Bytes.toBytes(longValue ^ INVERT_SIGN_MASK); } case BINARY: case STRING: { return Arrays.copyOf(value.array(), columnSchema.getMaxLength()); } default: return value.array(); } } finally { value.rewind(); // rewind the ByteBuffer's index pointer } }
/** * Return the name of the auto increment column in the table, or null. * * @return ColumnSchema with an auto increment modifier */ public String getAutoIncrementColumn() { for (ColumnSchema entry : columns) { if (entry.getIsAutoIncrement()) { return entry.getColumnName(); } } return null; }
@Test(expected = NullPointerException.class) public void testDeserializeInvalidColumnName() { ColumnSchema.deserialize(TEST_COL_SCHEMA.serialize(), null); }
checkNotNull(schema, "Column metadata is null."); ColumnType type = schema.getType(); if (schema.getIsNullable()) { return null; "yyyyMMdd HHmmss"); case DECIMAL: return extractDecimal(val, schema.getPrecision(), schema.getScale()); case STRING: case BINARY:
public FieldGenerator(ColumnSchema schema) { switch (schema.getType()) { case ULONG: case LONG: case TIME: { generator = new LongBytesGenerator(); break; } case DOUBLE: { generator = new DoubleBytesGenerator(); break; } case BINARY: { generator = CombinedGenerators.byteArrays( PrimitiveGenerators.integers(0, schema.getMaxLength())); break; } case STRING: { generator = new StringBytesGenerator(schema.getMaxLength()); break; } default: generator = CombinedGenerators.byteArrays( PrimitiveGenerators.fixedValues(32)); } if (schema.getIsNullable()) { generator = CombinedGenerators.nullsAnd(generator, 10); } }
@Test public void testDeserializeValidSerializedSchemaAndColumnName() { final ColumnSchema actualSchema = ColumnSchema.deserialize(TEST_COL_SCHEMA.serialize(), TEST_COL_SCHEMA.getColumnName()); assertEquals(TEST_COL_SCHEMA.getColumnName(), actualSchema.getColumnName()); assertEquals(TEST_COL_SCHEMA, actualSchema); }
/** * Construct a table schema from a map of column schemas and index schemas. * * @param columns Column schemas map [Not null] * @param indices Index schema map [Not null] */ public TableSchema(Collection<ColumnSchema> columns, Collection<IndexSchema> indices) { checkArgument(columns.size() > 0, "Table must have at least one column."); checkNotNull(indices); Map<String, AvroColumnSchema> columnSchemaMap = Maps.newHashMap(); for (ColumnSchema columnSchema : columns) { columnSchemaMap.put(columnSchema.getColumnName(), columnSchema.getAvroValue()); } Map<String, AvroIndexSchema> indexSchemaMap = Maps.newHashMap(); for (IndexSchema indexSchema : indices) { indexSchemaMap.put(indexSchema.getIndexName(), indexSchema.getAvroValue()); if (indexSchema.getIsUnique()) { uniqueIndexCount++; } } avroTableSchema = AvroTableSchema.newBuilder() .setColumns(columnSchemaMap) .setIndices(indexSchemaMap) .build(); this.columns = new LinkedList<ColumnSchema>(columns); this.indices = new LinkedList<IndexSchema>(indices); }
@Test public void testHasAutoIncrementColumnNotAutoInc() { final List<ColumnSchema> columns = ImmutableList.<ColumnSchema>of( ColumnSchema.builder(COLUMN_B, ColumnType.LONG).setIsAutoIncrement(false).build()); final TableSchema tableSchema = new TableSchema(columns, ImmutableList.<IndexSchema>of()); Verify.hasAutoIncrementColumn(tableSchema); } }
@Test(expected = NullPointerException.class) public void testDeserializeNullSerializedSchema() { ColumnSchema.deserialize(null, TEST_COLUMN); }
@Test(expected = IllegalArgumentException.class) public void testDeserializeEmptyColumnName() { ColumnSchema.deserialize(TEST_COL_SCHEMA.serialize(), ""); }
@Test public void testHasAutoIncrementColumn() { final List<ColumnSchema> columns = ImmutableList.<ColumnSchema>of( ColumnSchema.builder(COLUMN_B, ColumnType.LONG).setIsAutoIncrement(true).build()); final TableSchema tableSchema = new TableSchema(columns, ImmutableList.<IndexSchema>of()); Verify.hasAutoIncrementColumn(tableSchema); }
/** * Verifies that the index schema only index columns for columns that are available * @param indices A mapping of the index details, not null * @param columns A mapping of column details, not null * @throws NullPointerException Thrown if the indices or columns container is null * @throws IllegalArgumentException Thrown if a {@link IndexSchema} indexes * a column that is not an available column */ public static void isValidIndexSchema(final Collection<IndexSchema> indices, final Collection<ColumnSchema> columns) { checkNotNull(indices); checkNotNull(columns); Set<String> columnNames = Sets.newHashSet(); for (ColumnSchema column : columns) { columnNames.add(column.getColumnName()); } for (final IndexSchema index : indices) { for (final String column : index.getColumns()) { if (!columnNames.contains(column)) { throw new IllegalArgumentException("Only columns in the table may be indexed."); } } } } }
private TableSchema(AvroTableSchema avroTableSchema) { this.avroTableSchema = avroTableSchema; columns = Lists.newArrayList(); for (Map.Entry<String, AvroColumnSchema> entry : avroTableSchema.getColumns().entrySet()) { columns.add(new ColumnSchema(entry.getKey(), entry.getValue())); } indices = Lists.newArrayList(); for (Map.Entry<String, AvroIndexSchema> entry : avroTableSchema.getIndices().entrySet()) { indices.add(new IndexSchema(entry.getKey(), entry.getValue())); if (entry.getValue().getIsUnique()) { uniqueIndexCount++; } } }
@Test public void testIsValidTableSchemaValidSchema() { final List<ColumnSchema> columns = ImmutableList.<ColumnSchema>of( ColumnSchema.builder(COLUMN_B, ColumnType.LONG).setIsAutoIncrement(true).build()); Verify.isValidTableSchema(new TableSchema(columns, ImmutableList.<IndexSchema>of())); }
private Put putColumnIds(long tableId, Collection<ColumnSchema> columns) { long columnId = getNextColumnId(tableId, columns.size()); Put put = new Put(new ColumnsRowKey(tableId).encode()); for (ColumnSchema columnEntry : columns) { put.add(columnFamily, serializeName(columnEntry.getColumnName()), serializeId(columnId--)); } return put; }
/** * Deserialize a byte array into a column schema. * * @param serializedColumnSchema Byte array of the column schema * @param columnName Column name * @return An column schema for the byte array. */ public static ColumnSchema deserialize(byte[] serializedColumnSchema, String columnName) { checkNotNull(serializedColumnSchema); Verify.isNotNullOrEmpty(columnName); return new ColumnSchema(columnName, Util.deserializeAvroObject(serializedColumnSchema, reader)); }
@Override public ColumnSchema next() { ColumnType type = typeGen.next(); ColumnSchema.Builder builder = ColumnSchema.builder(MYSQL_NAME_GEN.next(), type); switch (type) { case STRING: case BINARY: builder.setMaxLength(lengthGen.next()); break; case LONG: case ULONG: case DOUBLE: builder.setIsAutoIncrement(RAND.nextBoolean()); break; case DECIMAL: int precision = RAND.nextInt(66); int scale = RAND.nextInt(Math.max(31, precision)); builder.setPrecision(precision).setScale(scale); break; default: break; } return builder.build(); } }
public RowGenerator(TableSchema schema) { super(); ImmutableMap.Builder<String, Generator<ByteBuffer>> recordGenerators = ImmutableMap.builder(); for (ColumnSchema column : schema.getColumns()) { recordGenerators.put(column.getColumnName(), new FieldGenerator(column)); } this.recordGenerators = recordGenerators.build(); }