public String partitionToPath(StructLike data) { StringBuilder sb = new StringBuilder(); Class<?>[] javaClasses = javaClasses(); for (int i = 0; i < javaClasses.length; i += 1) { PartitionField field = fields[i]; String valueString = field.transform().toHumanString(get(data, i, javaClasses[i])); if (i > 0) { sb.append("/"); } sb.append(field.name()).append("=").append(escape(valueString)); } return sb.toString(); }
private static PartitionData copyPartitionData(PartitionSpec spec, StructLike partitionData, PartitionData reuse) { PartitionData data = reuse; if (data == null) { data = newPartitionData(spec); } Class<?>[] javaClasses = spec.javaClasses(); List<PartitionField> fields = spec.fields(); for (int i = 0; i < fields.size(); i += 1) { data.set(i, partitionData.get(i, javaClasses[i])); } return data; }
PartitionSummary(PartitionSpec spec) { this.javaClasses = spec.javaClasses(); this.fields = new PartitionFieldStats[javaClasses.length]; List<Types.NestedField> partitionFields = spec.partitionType().fields(); for (int i = 0; i < fields.length; i += 1) { this.fields[i] = new PartitionFieldStats<>(partitionFields.get(i).type()); } }
PartitionRowConverter(Schema partitionSchema, PartitionSpec spec) { StructType partitionType = SparkSchemaUtil.convert(partitionSchema); StructField[] fields = partitionType.fields(); this.types = new DataType[fields.length]; this.positions = new int[types.length]; this.javaTypes = new Class<?>[types.length]; this.reusedRow = new GenericInternalRow(types.length); List<PartitionField> partitionFields = spec.fields(); for (int rowIndex = 0; rowIndex < fields.length; rowIndex += 1) { this.types[rowIndex] = fields[rowIndex].dataType(); int sourceId = partitionSchema.columns().get(rowIndex).fieldId(); for (int specIndex = 0; specIndex < partitionFields.size(); specIndex += 1) { PartitionField field = spec.fields().get(specIndex); if (field.sourceId() == sourceId && "identity".equals(field.transform().toString())) { positions[rowIndex] = specIndex; javaTypes[rowIndex] = spec.javaClasses()[specIndex]; break; } } } }