/** * Convert schema to binary */ public static byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); DataOutput dataOutput = new DataOutputStream(stream); dataOutput.writeShort(columnSchemas.size()); for (ColumnSchema columnSchema : columnSchemas) { if (columnSchema.getColumnReferenceId() == null) { columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId()); } columnSchema.write(dataOutput); } byte[] byteArray = stream.toByteArray(); // Compress to reduce the size of schema return CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor().compressByte( byteArray); }
/** * Read and return the schema in the header */ public List<ColumnSchema> readSchema() throws IOException { FileHeader fileHeader = readHeader(); List<ColumnSchema> columnSchemaList = new ArrayList<>(); List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema(); for (org.apache.carbondata.format.ColumnSchema table_column : table_columns) { ColumnSchema col = thriftColumnSchemaToWrapperColumnSchema(table_column); col.setColumnReferenceId(col.getColumnUniqueId()); columnSchemaList.add(col); } return columnSchemaList; } }
public static CarbonTable buildTable( String tablePath, String tableName, Configuration configuration) throws IOException { TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, "null", "null"); CarbonFile carbonFile = getFirstIndexFile(FileFactory.getCarbonFile(tablePath, configuration)); if (carbonFile == null) { throw new RuntimeException("Carbon index file not exists."); } org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil .inferSchemaFromIndexFile(carbonFile.getPath(), tableName); List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); for (org.apache.carbondata.format.ColumnSchema thriftColumnSchema : tableInfo .getFact_table().getTable_columns()) { ColumnSchema columnSchema = thriftColumnSchemaToWrapperColumnSchema(thriftColumnSchema); if (columnSchema.getColumnReferenceId() == null) { columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId()); } columnSchemaList.add(columnSchema); } tableInfoInfer.getFactTable().setListOfColumns(columnSchemaList); return CarbonTable.buildFromTableInfo(tableInfoInfer); }
id.setEncodingList(encodings); id.setColumnUniqueId(UUID.randomUUID().toString()); id.setColumnReferenceId(id.getColumnUniqueId()); id.setDimensionColumn(true); id.setSchemaOrdinal(schemaOrdinal++); date.setColumnUniqueId(UUID.randomUUID().toString()); date.setDimensionColumn(true); date.setColumnReferenceId(date.getColumnUniqueId()); date.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(date.getColumnName())) { country.setColumnReferenceId(country.getColumnUniqueId()); columnSchemas.add(country); name.setColumnReferenceId(name.getColumnUniqueId()); columnSchemas.add(name); phonetype.setColumnReferenceId(phonetype.getColumnUniqueId()); columnSchemas.add(phonetype); serialname.setColumnReferenceId(serialname.getColumnUniqueId()); columnSchemas.add(serialname); ColumnSchema salary = new ColumnSchema(); salary.setColumnUniqueId(UUID.randomUUID().toString()); salary.setDimensionColumn(false); salary.setColumnReferenceId(salary.getColumnUniqueId()); salary.setSchemaOrdinal(schemaOrdinal++);
wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value()); wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible()); wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId()); wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal()); wrapperColumnSchema.setSortColumn(false);
ColumnSchema wrapperColumnSchema = new ColumnSchema(); wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id()); wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId()); wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name()); DataType dataType = thriftDataTypeToWrapperDataType(externalColumnSchema.data_type);
newColumn.setColumnReferenceId(newColumn.getColumnUniqueId()); newColumn .setEncodingList(createEncoding(field.getDataType(), isInvertedIdxColumn, isComplexChild));