private void indexFields() { ImmutableMap.Builder<String, NestedField> byNameBuilder = ImmutableMap.builder(); ImmutableMap.Builder<Integer, NestedField> byIdBuilder = ImmutableMap.builder(); for (NestedField field : fields) { byNameBuilder.put(field.name(), field); byIdBuilder.put(field.fieldId(), field); } this.fieldsByName = byNameBuilder.build(); this.fieldsById = byIdBuilder.build(); } }
@Override public UpdateSchema deleteColumn(String name) { Types.NestedField field = schema.findField(name); Preconditions.checkArgument(field != null, "Cannot delete missing column: %s", name); Preconditions.checkArgument(!adds.containsKey(field.fieldId()), "Cannot delete a column that has additions: %s", name); Preconditions.checkArgument(!updates.containsKey(field.fieldId()), "Cannot delete a column that has updates: %s", name); deletes.add(field.fieldId()); return this; }
@Override public Set<Integer> field(Types.NestedField field, Set<Integer> fieldResult) { if (fieldResult == null) { fieldIds.add(field.fieldId()); } return fieldIds; }
private int find(int fieldId, Types.StructType struct) { List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { if (fields.get(i).fieldId() == fieldId) { return i; } } throw new ValidationException( "Cannot find top-level field id %d in struct: %s", fieldId, struct); }
@Override public Map<Integer, Types.NestedField> list(Types.ListType list, Map<Integer, Types.NestedField> elementResult) { for (Types.NestedField field : list.fields()) { index.put(field.fieldId(), field); } return null; }
public Builder hour(String sourceName) { String name = sourceName + "_hour"; checkAndAddPartitionName(name); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), name, Transforms.hour(sourceColumn.type()))); return this; }
private static ResourceFieldSchema convert(Types.NestedField field) throws IOException { ResourceFieldSchema result = convert(field.type()); result.setName(field.name()); result.setDescription(format("FieldId: %s", field.fieldId())); return result; }
@Override public Map<String, Integer> list(Types.ListType list, Map<String, Integer> elementResult) { for (Types.NestedField field : list.fields()) { addField(field.name(), field.fieldId()); } return null; }
@Override public Map<Integer, Types.NestedField> map(Types.MapType map, Map<Integer, Types.NestedField> keyResult, Map<Integer, Types.NestedField> valueResult) { for (Types.NestedField field : map.fields()) { index.put(field.fieldId(), field); } return null; } }
public Builder truncate(String sourceName, int width) { String name = sourceName + "_trunc"; checkAndAddPartitionName(name); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), name, Transforms.truncate(sourceColumn.type(), width))); return this; }
public Builder bucket(String sourceName, int numBuckets) { String name = sourceName + "_bucket"; checkAndAddPartitionName(name); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), name, Transforms.bucket(sourceColumn.type(), numBuckets))); return this; }
public Builder identity(String sourceName) { checkAndAddPartitionName(sourceName); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), sourceName, Transforms.identity(sourceColumn.type()))); return this; }
@Override public Set<Integer> map(Types.MapType map, Set<Integer> keyResult, Set<Integer> valueResult) { if (valueResult == null) { for (Types.NestedField field : map.fields()) { fieldIds.add(field.fieldId()); } } return fieldIds; } }
@Override public Map<String, Integer> map(Types.MapType map, Map<String, Integer> keyResult, Map<String, Integer> valueResult) { for (Types.NestedField field : map.fields()) { addField(field.name(), field.fieldId()); } return null; }
public Builder month(String sourceName) { String name = sourceName + "_month"; checkAndAddPartitionName(name); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), name, Transforms.month(sourceColumn.type()))); return this; }
public Builder year(String sourceName) { String name = sourceName + "_year"; checkAndAddPartitionName(name); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), name, Transforms.year(sourceColumn.type()))); return this; }
@Override public Set<Integer> list(Types.ListType list, Set<Integer> elementResult) { if (elementResult == null) { for (Types.NestedField field : list.fields()) { fieldIds.add(field.fieldId()); } } return fieldIds; }
@Override public Map<Integer, Types.NestedField> field(Types.NestedField field, Map<Integer, Types.NestedField> fieldResult) { index.put(field.fieldId(), field); return null; }
public Builder day(String sourceName) { String name = sourceName + "_day"; checkAndAddPartitionName(name); Types.NestedField sourceColumn = findSourceColumn(sourceName); fields.add(new PartitionField( sourceColumn.fieldId(), name, Transforms.day(sourceColumn.type()))); return this; }