public static PartialRow toRangeBoundToPartialRow(Schema schema, RangePartitionDefinition definition, RangeBoundValue boundValue) { PartialRow partialRow = new PartialRow(schema); if (boundValue != null) { List<Integer> rangeColumns = definition.getColumns().stream() .map(schema::getColumnIndex).collect(toImmutableList()); if (rangeColumns.size() != boundValue.getValues().size()) { throw new IllegalStateException("Expected " + rangeColumns.size() + " range columns, but got " + boundValue.getValues().size()); } for (int i = 0; i < rangeColumns.size(); i++) { Object obj = boundValue.getValues().get(i); int idx = rangeColumns.get(i); ColumnSchema columnSchema = schema.getColumnByIndex(idx); setColumnValue(partialRow, idx, obj, columnSchema.getType(), columnSchema.getName()); } } return partialRow; }
public static Type fromKuduColumn(ColumnSchema column) { return fromKuduClientType(column.getType(), column.getTypeAttributes()); }
private ColumnMetadata getColumnMetadata(ColumnSchema column) { Map<String, Object> properties = new LinkedHashMap<>(); StringBuilder extra = new StringBuilder(); if (column.isKey()) { properties.put(KuduTableProperties.PRIMARY_KEY, true); extra.append("primary_key, "); } if (column.isNullable()) { properties.put(KuduTableProperties.NULLABLE, true); extra.append("nullable, "); } String encoding = KuduTableProperties.lookupEncodingString(column.getEncoding()); if (!column.getEncoding().equals(ColumnSchema.Encoding.AUTO_ENCODING)) { properties.put(KuduTableProperties.ENCODING, encoding); } extra.append("encoding=").append(encoding).append(", "); String compression = KuduTableProperties.lookupCompressionString(column.getCompressionAlgorithm()); if (!column.getCompressionAlgorithm().equals(ColumnSchema.CompressionAlgorithm.DEFAULT_COMPRESSION)) { properties.put(KuduTableProperties.COMPRESSION, compression); } extra.append("compression=").append(compression); Type prestoType = TypeHelper.fromKuduColumn(column); return new ColumnMetadata(column.getName(), prestoType, null, extra.toString(), false, properties); }
private ConnectorTableMetadata getTableMetadata(KuduTableHandle tableHandle) { KuduTable table = tableHandle.getTable(clientSession); Schema schema = table.getSchema(); List<ColumnMetadata> columnsMetaList = schema.getColumns().stream() .filter(column -> !column.isKey() || !column.getName().equals(KuduColumnHandle.ROW_ID)) .map(this::getColumnMetadata) .collect(toImmutableList()); Map<String, Object> properties = clientSession.getTableProperties(tableHandle); return new ConnectorTableMetadata(tableHandle.getSchemaTableName(), columnsMetaList, properties); }
/** {@inheritDoc} */ @Override public String toString() { int numCols = schema.getColumnCount(); StringBuilder sb = new StringBuilder(); sb.append('('); boolean first = true; for (int idx = 0; idx < numCols; ++idx) { if (!columnsBitSet.get(idx)) { continue; } if (first) { first = false; } else { sb.append(", "); } ColumnSchema col = schema.getColumnByIndex(idx); sb.append(col.getType().getName()); if (col.getTypeAttributes() != null) { sb.append(col.getTypeAttributes().toStringForType(col.getType())); } sb.append(' '); sb.append(col.getName()); sb.append('='); appendCellValueDebugString(idx, sb); } sb.append(')'); return sb.toString(); }
public static Common.ColumnSchemaPB columnToPb(Common.ColumnSchemaPB.Builder schemaBuilder, ColumnSchema column) { schemaBuilder .setName(column.getName()) .setType(column.getType().getDataType(column.getTypeAttributes())) .setIsKey(column.isKey()) .setIsNullable(column.isNullable()) .setCfileBlockSize(column.getDesiredBlockSize()); if (column.getEncoding() != null) { schemaBuilder.setEncoding(column.getEncoding().getInternalPbType()); } if (column.getCompressionAlgorithm() != null) { schemaBuilder.setCompression(column.getCompressionAlgorithm().getInternalPbType()); } if (column.getDefaultValue() != null) { schemaBuilder.setReadDefaultValue(UnsafeByteOperations.unsafeWrap( objectToWireFormat(column, column.getDefaultValue()))); } if(column.getTypeAttributes() != null) { schemaBuilder.setTypeAttributes( columnTypeAttributesToPb(Common.ColumnTypeAttributesPB.newBuilder(), column)); } return schemaBuilder.build(); }
@Override public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle connectorTableHandle) { KuduTableHandle tableHandle = (KuduTableHandle) connectorTableHandle; Schema schema = clientSession.getTableSchema(tableHandle); ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder(); for (int ordinal = 0; ordinal < schema.getColumnCount(); ordinal++) { ColumnSchema col = schema.getColumnByIndex(ordinal); String name = col.getName(); Type type = TypeHelper.fromKuduColumn(col); KuduColumnHandle columnHandle = new KuduColumnHandle(name, ordinal, type); columnHandles.put(name, columnHandle); } return columnHandles.build(); }
@Override public RelDataType getRowType(RelDataTypeFactory typeFactory) { List<String> names = Lists.newArrayList(); List<RelDataType> types = Lists.newArrayList(); for (ColumnSchema column : schema.getColumns()) { names.add(column.getName()); RelDataType type = getSqlTypeFromKuduType(typeFactory, column.getType()); type = typeFactory.createTypeWithNullability(type, column.isNullable()); types.add(type); } return typeFactory.createStructType(types, names); }
private static Object toValue(Schema schema, PartialRow bound, Integer idx) { Type type = schema.getColumnByIndex(idx).getType(); switch (type) { case UNIXTIME_MICROS: long millis = bound.getLong(idx) / 1000; return ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC).print(millis); case STRING: return bound.getString(idx); case INT64: return bound.getLong(idx); case INT32: return bound.getInt(idx); case INT16: return bound.getShort(idx); case INT8: return (short) bound.getByte(idx); case BOOL: return bound.getBoolean(idx); case BINARY: return bound.getBinaryCopy(idx); default: throw new IllegalStateException("Unhandled type " + type + " for range partition"); } }
if (columnSchema.getType() == Type.INT32) { defaultValue = defaultInt; } else if (columnSchema.getType() == Type.BOOL) { defaultValue = true; } else { new ColumnSchema.ColumnSchemaBuilder(columnSchema.getName(), columnSchema.getType()) .key(columnSchema.isKey()) .nullable(columnSchema.isNullable()) .defaultValue(defaultValue).build()); assertEquals(defaultInt, kuduTable.getSchema().getColumnByIndex(0).getDefaultValue()); assertEquals(defaultString, kuduTable.getSchema().getColumnByIndex(columns.size() - 2).getDefaultValue()); assertEquals(true, kuduTable.getSchema().getColumnByIndex(columns.size() - 1).getDefaultValue()); assertNull(kuduTableWithoutDefaults.getSchema().getColumnByIndex(0).getDefaultValue()); createTableWithSplitsAndTest(splitTablePrefix, 3); createTableWithSplitsAndTest(splitTablePrefix, 10);
if (column.isKey()) { primaryKeyColumns.add(column); hasNulls |= column.isNullable(); columnOffsets[index] = offset; offset += column.getTypeSize(); if (this.columnsByName.put(column.getName(), index) != null) { throw new IllegalArgumentException( String.format("Column names must be unique: %s", columns)); if (column.getType() == Type.STRING || column.getType() == Type.BINARY) { varLenCnt++;
/** * Clone the given column schema instance. The new instance will include only the name, type, and * nullability of the passed one. * @return a new column schema */ private static ColumnSchema getStrippedColumnSchema(ColumnSchema columnToClone) { return new ColumnSchema.ColumnSchemaBuilder(columnToClone.getName(), columnToClone.getType()) .nullable(columnToClone.isNullable()) .typeAttributes(columnToClone.getTypeAttributes()) .build(); }
Preconditions.checkArgument(a.getSchema().getColumnByIndex(index).isKey()); Preconditions.checkArgument(a.isSet(index)); Preconditions.checkArgument(b.isSet(index)); Type type = column.getType(); int offset = a.getSchema().getColumnOffset(index); return Bytes.getDouble(a.rowAlloc, offset) == Bytes.getDouble(b.rowAlloc, offset); case DECIMAL: ColumnTypeAttributes typeAttributes = column.getTypeAttributes(); int precision = typeAttributes.getPrecision(); int scale = typeAttributes.getScale();
private void setNull(ColumnSchema column) { assert nullsBitSet != null; checkNotFrozen(); checkColumnExists(column); if (!column.isNullable()) { throw new IllegalArgumentException(column.getName() + " cannot be set to null"); } int idx = schema.getColumns().indexOf(column); columnsBitSet.set(idx); nullsBitSet.set(idx); }
/** * Encodes a single column of a row into the output buffer. * @param row the row being encoded * @param columnIdx the column index of the column to encode * @param isLast whether the column is the last component of the key */ private static void encodeColumn(PartialRow row, int columnIdx, boolean isLast, ByteVec buf) { final Schema schema = row.getSchema(); final ColumnSchema column = schema.getColumnByIndex(columnIdx); if (!row.isSet(columnIdx)) { throw new IllegalStateException(String.format("Primary key column %s is not set", column.getName())); } final Type type = column.getType(); if (type == Type.STRING || type == Type.BINARY) { encodeBinary(row.getVarLengthData().get(columnIdx), isLast, buf); } else { encodeSignedInt(row.getRowAlloc(), schema.getColumnOffset(columnIdx), column.getTypeSize(), buf); } }
switch (pci.kuduColumn.getType()) { case BINARY: { ByteBuffer value = result.getBinary(pci.index); if (pci.kuduColumn.isNullable()) { ((NullableVarBinaryVector.Mutator) pci.vv.getMutator()) .setSafe(rowIndex, value, 0, value.remaining()); if (pci.kuduColumn.isNullable()) { ((NullableVarCharVector.Mutator) pci.vv.getMutator()) .setSafe(rowIndex, value, 0, value.remaining()); if (pci.kuduColumn.isNullable()) { ((NullableBitVector.Mutator) pci.vv.getMutator()) .setSafe(rowIndex, result.getBoolean(pci.index) ? 1 : 0); if (pci.kuduColumn.isNullable()) { ((NullableFloat8Vector.Mutator) pci.vv.getMutator()) .setSafe(rowIndex, result.getDouble(pci.index)); if (pci.kuduColumn.isNullable()) { ((NullableFloat4Vector.Mutator) pci.vv.getMutator()) .setSafe(rowIndex, result.getFloat(pci.index)); if (pci.kuduColumn.isNullable()) { ((NullableIntVector.Mutator) pci.vv.getMutator()) .setSafe(rowIndex, result.getShort(pci.index)); if (pci.kuduColumn.isNullable()) { ((NullableIntVector.Mutator) pci.vv.getMutator())
@Test public void testAlterModifyColumns() throws Exception { KuduTable table = createTable(ImmutableList.<Pair<Integer,Integer>>of()); insertRows(table, 0, 100); assertEquals(100, countRowsInTable(table)); // Check for expected defaults. ColumnSchema col = table.getSchema().getColumns().get(1); assertEquals(CompressionAlgorithm.DEFAULT_COMPRESSION, col.getCompressionAlgorithm()); assertEquals(Encoding.AUTO_ENCODING, col.getEncoding()); assertEquals(null, col.getDefaultValue()); // Alter the table. client.alterTable(tableName, new AlterTableOptions() .changeCompressionAlgorithm(col.getName(), CompressionAlgorithm.SNAPPY) .changeEncoding(col.getName(), Encoding.RLE) .changeDefault(col.getName(), 0)); // Check for new values. table = client.openTable(tableName); col = table.getSchema().getColumns().get(1); assertEquals(CompressionAlgorithm.SNAPPY, col.getCompressionAlgorithm()); assertEquals(Encoding.RLE, col.getEncoding()); assertEquals(0, col.getDefaultValue()); }
int columnIdx = table.getSchema().getColumnIndex(column.getName()); ColumnSchema schema = table.getSchema().getColumnByIndex(columnIdx); if (column.getType() != schema.getType().getDataType(schema.getTypeAttributes())) { throw new IllegalStateException(String.format( "invalid type %s for column '%s' in scan token, expected: %s", column.getType().name(), column.getName(), schema.getType().name())); if (column.getIsNullable() != schema.isNullable()) { throw new IllegalStateException(String.format( "invalid nullability for column '%s' in scan token, expected: %s",
Schema schema = row.getSchema(); ColumnSchema column = schema.getColumnByIndex(idx); switch (column.getType()) { case INT8: row.addByte(idx, (byte) (buf.get() ^ Byte.MIN_VALUE)); int scale = column.getTypeAttributes().getScale(); int size = column.getTypeSize(); switch (size) { case DecimalUtil.DECIMAL32_SIZE: throw new IllegalArgumentException(String.format( "The column type %s is not a valid key component type", schema.getColumnByIndex(idx).getType()));
Type type = column.getType(); switch (type) { case BOOL: case DOUBLE: case DECIMAL: { Preconditions.checkArgument(value.length == column.getTypeSize()); System.arraycopy(value, 0, rowAlloc, getPositionInRowAllocAndSetBitSet(index), value.length);