/** * @return column unique id */ public String getColumnId() { return columnSchema.getColumnUniqueId(); }
@Override public int compare(ColumnSchema o1, ColumnSchema o2) { return o1.getColumnUniqueId().compareTo(o2.getColumnUniqueId()); } });
/** * hash code * @return */ @Override public int hashCode() { return this.getColumnSchema().getColumnUniqueId().hashCode(); } }
@Override public boolean accept(CarbonFile path) { if (path.getName().startsWith(columnSchema.getColumnUniqueId())) { return true; } return false; } });
/** * This method will add the prepare the max column cardinality map * * @param columnCardinalityMap * @param currentBlockSchema * @param currentBlockCardinality */ public static void addColumnCardinalityToMap(Map<String, Integer> columnCardinalityMap, List<ColumnSchema> currentBlockSchema, int[] currentBlockCardinality) { for (int i = 0; i < currentBlockCardinality.length; i++) { // add value to map only if does not exist or new cardinality is > existing value String columnUniqueId = currentBlockSchema.get(i).getColumnUniqueId(); Integer value = columnCardinalityMap.get(columnUniqueId); if (null == value) { columnCardinalityMap.put(columnUniqueId, currentBlockCardinality[i]); } else { if (currentBlockCardinality[i] > value) { columnCardinalityMap.put(columnUniqueId, currentBlockCardinality[i]); } } } }
/** * Convert schema to binary */ public static byte[] convertSchemaToBinary(List<ColumnSchema> columnSchemas) throws IOException { ByteArrayOutputStream stream = new ByteArrayOutputStream(); DataOutput dataOutput = new DataOutputStream(stream); dataOutput.writeShort(columnSchemas.size()); for (ColumnSchema columnSchema : columnSchemas) { if (columnSchema.getColumnReferenceId() == null) { columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId()); } columnSchema.write(dataOutput); } byte[] byteArray = stream.toByteArray(); // Compress to reduce the size of schema return CompressorFactory.NativeSupportedCompressor.SNAPPY.getCompressor().compressByte( byteArray); }
/** * Read and return the schema in the header */ public List<ColumnSchema> readSchema() throws IOException { FileHeader fileHeader = readHeader(); List<ColumnSchema> columnSchemaList = new ArrayList<>(); List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema(); for (org.apache.carbondata.format.ColumnSchema table_column : table_columns) { ColumnSchema col = thriftColumnSchemaToWrapperColumnSchema(table_column); col.setColumnReferenceId(col.getColumnUniqueId()); columnSchemaList.add(col); } return columnSchemaList; } }
if (carbonDimension.getColumnSchema().getColumnUniqueId() .equalsIgnoreCase(queryColumn.getColumnId())) { return true;
public static CarbonTable buildTable( String tablePath, String tableName, Configuration configuration) throws IOException { TableInfo tableInfoInfer = CarbonUtil.buildDummyTableInfo(tablePath, "null", "null"); CarbonFile carbonFile = getFirstIndexFile(FileFactory.getCarbonFile(tablePath, configuration)); if (carbonFile == null) { throw new RuntimeException("Carbon index file not exists."); } org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil .inferSchemaFromIndexFile(carbonFile.getPath(), tableName); List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); for (org.apache.carbondata.format.ColumnSchema thriftColumnSchema : tableInfo .getFact_table().getTable_columns()) { ColumnSchema columnSchema = thriftColumnSchemaToWrapperColumnSchema(thriftColumnSchema); if (columnSchema.getColumnReferenceId() == null) { columnSchema.setColumnReferenceId(columnSchema.getColumnUniqueId()); } columnSchemaList.add(columnSchema); } tableInfoInfer.getFactTable().setListOfColumns(columnSchemaList); return CarbonTable.buildFromTableInfo(tableInfoInfer); }
LOGGER.error("Failed to delete dictionary or sortIndex file for column " + columnSchema.getColumnName() + "with column ID " + columnSchema.getColumnUniqueId()); removeDictionaryColumnFromCache(identifier, columnSchema.getColumnUniqueId());
id.setEncodingList(encodings); id.setColumnUniqueId(UUID.randomUUID().toString()); id.setColumnReferenceId(id.getColumnUniqueId()); id.setDimensionColumn(true); id.setSchemaOrdinal(schemaOrdinal++); date.setColumnUniqueId(UUID.randomUUID().toString()); date.setDimensionColumn(true); date.setColumnReferenceId(date.getColumnUniqueId()); date.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(date.getColumnName())) { country.setColumnReferenceId(country.getColumnUniqueId()); columnSchemas.add(country); name.setColumnReferenceId(name.getColumnUniqueId()); columnSchemas.add(name); phonetype.setColumnReferenceId(phonetype.getColumnUniqueId()); columnSchemas.add(phonetype); serialname.setColumnReferenceId(serialname.getColumnUniqueId()); columnSchemas.add(serialname); ColumnSchema salary = new ColumnSchema(); salary.setColumnUniqueId(UUID.randomUUID().toString()); salary.setDimensionColumn(false); salary.setColumnReferenceId(salary.getColumnUniqueId()); salary.setSchemaOrdinal(schemaOrdinal++);
wrapperColumnSchema.getDataType()), wrapperColumnSchema.getColumnName(), wrapperColumnSchema.getColumnUniqueId(), true, encoders,
newColumn.setColumnReferenceId(newColumn.getColumnUniqueId()); newColumn .setEncodingList(createEncoding(field.getDataType(), isInvertedIdxColumn, isComplexChild));