private static Map<Integer, PartitionSpec> indexSpecs(List<PartitionSpec> specs) { ImmutableMap.Builder<Integer, PartitionSpec> builder = ImmutableMap.builder(); for (PartitionSpec spec : specs) { builder.put(spec.specId(), spec); } return builder.build(); } }
public static void toJson(PartitionSpec spec, JsonGenerator generator) throws IOException { generator.writeStartObject(); generator.writeNumberField(SPEC_ID, spec.specId()); generator.writeFieldName(FIELDS); toJsonFields(spec, generator); generator.writeEndObject(); }
public TableMetadata updatePartitionSpec(PartitionSpec partitionSpec) { PartitionSpec.checkCompatibility(partitionSpec, schema); // if the spec already exists, use the same ID. otherwise, use 1 more than the highest ID. int newDefaultSpecId = INITIAL_SPEC_ID; for (PartitionSpec spec : specs) { if (partitionSpec.compatibleWith(spec)) { newDefaultSpecId = spec.specId(); break; } else if (newDefaultSpecId <= spec.specId()) { newDefaultSpecId = spec.specId() + 1; } } Preconditions.checkArgument(defaultSpecId != newDefaultSpecId, "Cannot set default partition spec to the current default"); ImmutableList.Builder<PartitionSpec> builder = ImmutableList.<PartitionSpec>builder() .addAll(specs); if (!specsById.containsKey(newDefaultSpecId)) { // get a fresh spec to ensure the spec ID is set to the new default builder.add(freshSpec(newDefaultSpecId, schema, partitionSpec)); } return new TableMetadata(ops, null, location, System.currentTimeMillis(), lastColumnId, schema, newDefaultSpecId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog); }
for (PartitionSpec spec : specs) { if (freshSpec.compatibleWith(spec)) { specId = spec.specId(); break;
ManifestWriter(PartitionSpec spec, OutputFile file, long snapshotId) { this.location = file.location(); this.file = file; this.specId = spec.specId(); this.writer = newAppender(FileFormat.AVRO, spec, file); this.snapshotId = snapshotId; this.reused = new ManifestEntry(spec.partitionType()); this.stats = new PartitionSummary(spec); }
private static <D> FileAppender<D> newAppender(FileFormat format, PartitionSpec spec, OutputFile file) { Schema manifestSchema = ManifestEntry.getSchema(spec.partitionType()); try { switch (format) { case AVRO: return Avro.write(file) .schema(manifestSchema) .named("manifest_entry") .meta("schema", SchemaParser.toJson(spec.schema())) .meta("partition-spec", PartitionSpecParser.toJsonFields(spec)) .meta("partition-spec-id", String.valueOf(spec.specId())) .build(); default: throw new IllegalArgumentException("Unsupported format: " + format); } } catch (IOException e) { throw new RuntimeIOException(e, "Failed to create manifest writer for path: " + file); } } }
Snapshot previousSnapshot = new BaseSnapshot( ops, previousSnapshotId, null, previousSnapshotId, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), spec.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops, currentSnapshotId, previousSnapshotId, currentSnapshotId, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId())));
Snapshot previousSnapshot = new BaseSnapshot( ops, previousSnapshotId, null, previousSnapshotId, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), spec.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( ops, currentSnapshotId, previousSnapshotId, currentSnapshotId, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId()))); metadata.specs().get(0).compatibleWith(spec)); Assert.assertEquals("PartitionSpec should have ID TableMetadata.INITIAL_SPEC_ID", TableMetadata.INITIAL_SPEC_ID, metadata.specs().get(0).specId()); Assert.assertEquals("Properties should match", expected.properties(), metadata.properties());
Snapshot previousSnapshot = new BaseSnapshot( null, previousSnapshotId, null, previousSnapshotId, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.1.avro"), spec.specId()))); long currentSnapshotId = System.currentTimeMillis(); Snapshot currentSnapshot = new BaseSnapshot( null, currentSnapshotId, previousSnapshotId, currentSnapshotId, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId())));