private void checkRepeatColumnName(StructField field) { for (ColumnSchema column : sortColumns) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : dimension) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : complex) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } for (ColumnSchema column : measures) { if (column.getColumnName().equalsIgnoreCase(field.getFieldName())) { throw new IllegalArgumentException("column name already exists"); } } }
private static void setFileHeader(Configuration configuration, CarbonLoadModel model) throws IOException { StructType inputSchema = getInputSchema(configuration); if (inputSchema == null || inputSchema.getFields().size() == 0) { throw new UnsupportedOperationException("Input schema must be set"); } List<StructField> fields = inputSchema.getFields(); StringBuilder builder = new StringBuilder(); String[] columns = new String[fields.size()]; int i = 0; for (StructField field : fields) { builder.append(field.getFieldName()); builder.append(","); columns[i++] = field.getFieldName(); } String header = builder.toString(); model.setCsvHeader(header.substring(0, header.length() - 1)); model.setCsvHeaderColumns(columns); }
/** * Throw exception if {@param field} name is repeated */ private void checkRepeatColumnName(StructField field, String parentName) { checkRepeatColumnName( new StructField(parentName + "." + field.getFieldName(), field.getDataType(), field.getChildren())); }
List<StructField> fields = ((StructType) dataType).getFields(); for (StructField field : fields) { structFields.add(prepareSubFields(field.getFieldName(), field.getDataType()));
for (StructField childFld : field.getChildren()) { structFieldsArray .add(new StructField(childFld.getFieldName(), childFld.getDataType()));
newColumn.setColumnName(parentName + "." + field.getFieldName()); } else { newColumn.setColumnName(field.getFieldName()); newColumn.setColumnUniqueId(field.getFieldName()); newColumn.setColumnReferenceId(newColumn.getColumnUniqueId()); newColumn String colName = getColNameForArray(valIndex); addColumn(new StructField(colName, ((ArrayType) field.getDataType()).getElementType()), field.getFieldName(), valIndex, false, true, isInvertedIdxColumn); } else if (DataTypes.isStructType(field.getDataType()) && ((StructType) field.getDataType()).getFields().size() > 0) {