/** * @return the colName */ public String getColName() { return columnSchema.getColumnName(); }
/** * returns whether column is complex column based on column name for child column * @return */ public boolean isComplexColumn() { return this.getColumnName() .contains(".val") || this.getColumnName().contains("."); } }
/** * In case of non transactional table just set columnuniqueid as columnName to support * backward compatabiity. non transactional tables column uniqueid is always equal to * columnname */ public static void updateColumnUniqueIdForNonTransactionTable(List<ColumnSchema> columnSchemas) { for (ColumnSchema columnSchema : columnSchemas) { // In case of complex types only add the name after removing parent names. int index = columnSchema.getColumnName().lastIndexOf("."); if (index >= 0) { columnSchema.setColumnUniqueId(columnSchema.getColumnName() .substring(index + 1, columnSchema.getColumnName().length())); } else { columnSchema.setColumnUniqueId(columnSchema.getColumnName()); } } }
/** * Project all Columns for carbon reader * * @return String araay of columnNames * @param carbonTable */ public String[] projectAllColumns(CarbonTable carbonTable) { List<ColumnSchema> colList = carbonTable.getTableInfo().getFactTable().getListOfColumns(); List<String> projectColumns = new ArrayList<>(); // complex type and add just the parent column name while skipping the child columns. for (ColumnSchema col : colList) { if (!col.getColumnName().contains(".")) { projectColumns.add(col.getColumnName()); } } return projectColumns.toArray(new String[projectColumns.size()]); } }
private void checkRepeatColumnName(StructField field) { for (ColumnSchema column : sortColumns) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : dimension) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : complex) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : measures) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } }
/** * Read schema from carbondata file * * @param dataFilePath carbondata file path * @return carbon data schema * @throws IOException */ public static Schema readSchemaFromDataFile(String dataFilePath) throws IOException { CarbonHeaderReader reader = new CarbonHeaderReader(dataFilePath); List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>(); List<ColumnSchema> schemaList = reader.readSchema(); for (int i = 0; i < schemaList.size(); i++) { ColumnSchema columnSchema = schemaList.get(i); if (!(columnSchema.getColumnName().contains("."))) { columnSchemaList.add(columnSchema); } } return new Schema(columnSchemaList); }
/** * Below method will be used to get the column schema based on parent column name * @param columName * parent column name * @return child column schema */ public ColumnSchema getChildColByParentColName(String columName) { List<ColumnSchema> listOfColumns = childSchema.getListOfColumns(); for (ColumnSchema columnSchema : listOfColumns) { List<ParentColumnTableRelation> parentColumnTableRelations = columnSchema.getParentColumnTableRelations(); if (null != parentColumnTableRelations && parentColumnTableRelations.size() == 1 && parentColumnTableRelations.get(0).getColumnName().equalsIgnoreCase(columName) && columnSchema.getColumnName().endsWith(columName)) { return columnSchema; } } return null; }
/** * Construct Field from ColumnSchema * * @param columnSchema ColumnSchema, Store the information about the column meta data */ public Field(ColumnSchema columnSchema) { this.name = columnSchema.getColumnName(); this.type = columnSchema.getDataType(); children = new LinkedList<>(); schemaOrdinal = columnSchema.getSchemaOrdinal(); precision = columnSchema.getPrecision(); scale = columnSchema.getScale(); }
/** * Below method will be used to get column schema for projection and * filter query column * * @param queryColumn query column * @param aggregationDataMapSchema selected data map schema * @return column schema */ private ColumnSchema getColumnSchema(QueryColumn queryColumn, AggregationDataMapSchema aggregationDataMapSchema) { ColumnSchema columnSchemaByParentName = null; if (!queryColumn.getTimeseriesFunction().isEmpty()) { columnSchemaByParentName = aggregationDataMapSchema .getTimeseriesChildColBasedByParent(queryColumn.getColumnSchema().getColumnName(), queryColumn.getTimeseriesFunction()); } else { columnSchemaByParentName = aggregationDataMapSchema .getNonAggNonTimeseriesChildColBasedByParent( queryColumn.getColumnSchema().getColumnName()); } return columnSchemaByParentName; } }
columnLocalDictGenMap.put(columnSchema.getColumnName(), new ColumnLocalDictionaryGenerator(localDictionaryThreshold, columnSchema.getDataType() == DataTypes.VARCHAR ?
/** * initialize partitioner for bucket column */ private void initializeBucketColumnPartitioner() { List<Integer> indexes = new ArrayList<>(); List<ColumnSchema> columnSchemas = new ArrayList<>(); DataField[] inputDataFields = getOutput(); BucketingInfo bucketingInfo = configuration.getBucketingInfo(); for (int i = 0; i < inputDataFields.length; i++) { for (int j = 0; j < bucketingInfo.getListOfColumns().size(); j++) { if (inputDataFields[i].getColumn().getColName() .equals(bucketingInfo.getListOfColumns().get(j).getColumnName())) { indexes.add(i); columnSchemas.add(inputDataFields[i].getColumn().getColumnSchema()); break; } } } // hash partitioner to dispatch rows by bucket column this.partitioner = new HashPartitionerImpl(indexes, columnSchemas, bucketingInfo.getNumOfRanges()); }
DataType dataType = carbonColumn.getDataType(); if (org.apache.carbondata.core.metadata.datatype.DataTypes.isDecimal(dataType)) { fields.add(new StructField(carbonColumn.getColumnName(), new DecimalType(carbonColumn.getPrecision(), carbonColumn.getScale()), true, Metadata.empty())); fields.add( new StructField( carbonColumn.getColumnName(), CarbonMetastoreTypes.toDataType( String.format("struct<%s>", SparkTypeConverter.getStructChildren(table, carbonColumn.getColumnName()))), true, Metadata.empty())); fields.add( new StructField( carbonColumn.getColumnName(), CarbonMetastoreTypes.toDataType( String.format("array<%s>", SparkTypeConverter.getArrayChildren( table, carbonColumn.getColumnName()))), true, Metadata.empty())); } else { fields.add(new StructField(carbonColumn.getColumnName(), convertCarbonToSparkDataType(carbonColumn.getDataType()), true, Metadata.empty()));
if (!Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName()) && column.getDataType().isComplexType()) { ordinal = i + 1; Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName()) && column.getDataType().isComplexType()) { ordinal = i + 1; || column.getDataType().equals(DataTypes.VARCHAR)) && !column .hasEncoding(Encoding.DICTIONARY)) { if (!Arrays.asList(listOfDictionaryExcludeColumns).contains(column.getColumnName())) { column.setLocalDictColumn(true); if (localDictIncludeColumns.contains(column.getColumnName()) && column.getDataType() .isComplexType()) { ordinal = i + 1; !column.hasEncoding(Encoding.DICTIONARY) && localDictIncludeColumns.toLowerCase() .contains(column.getColumnName().toLowerCase())) { for (String dictColumn : listOfDictionaryIncludeColumns) { if (dictColumn.trim().equalsIgnoreCase(column.getColumnName())) { column.setLocalDictColumn(true);
id.setDimensionColumn(true); id.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(id.getColumnName())) { id.setSortColumn(true); date.setColumnReferenceId(date.getColumnUniqueId()); date.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(date.getColumnName())) { date.setSortColumn(true); country.setSortColumn(true); country.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(country.getColumnName())) { country.setSortColumn(true); name.setDimensionColumn(true); name.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(name.getColumnName())) { name.setSortColumn(true); phonetype.setDimensionColumn(true); phonetype.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(phonetype.getColumnName())) { phonetype.setSortColumn(true); serialname.setDimensionColumn(true); serialname.setSchemaOrdinal(schemaOrdinal++); if (sortColumns.contains(serialname.getColumnName())) { serialname.setSortColumn(true);
} catch (IOException e) { LOGGER.error("Failed to delete dictionary or sortIndex file for column " + columnSchema.getColumnName() + "with column ID " + columnSchema.getColumnUniqueId());
fromWrapperToExternalDataType( wrapperColumnSchema.getDataType()), wrapperColumnSchema.getColumnName(), wrapperColumnSchema.getColumnUniqueId(), true,
String parentFieldName = newColumn.getColumnName(); if (DataTypes.isArrayType(field.getDataType())) { String colName = getColNameForArray(valIndex);
PartitionInfo partitionInfo) { ExpressionType filterExpressionType = expressionTree.getFilterExpressionType(); String partitionColumnName = partitionInfo.getColumnSchemaList().get(0).getColumnName(); BinaryExpression currentExpression = null; ColumnExpression left = null;