@Override public Map<String, String> getInputFormatConfiguration() { Collection<PartitionKey> inputKeys = getInputKeys(); List<Location> inputLocations = new ArrayList<>(inputKeys.size()); Map<String, PartitionKey> pathToKey = new HashMap<>(inputKeys.size()); for (PartitionKey key : inputKeys) { PartitionDetail partition = getPartition(key); String path = Objects.requireNonNull(partition).getRelativePath(); Location partitionLocation = files.getLocation(path); inputLocations.add(partitionLocation); pathToKey.put(partitionLocation.toURI().toString(), key); } Map<String, String> inputFormatConfiguration = files.getInputFormatConfiguration(inputLocations); inputFormatConfiguration.put(PATH_TO_PARTITIONING_MAPPING, GSON.toJson(pathToKey)); return inputFormatConfiguration; }
@Override public Map<String, String> getInputFormatConfiguration() { Collection<PartitionKey> inputKeys = getInputKeys(); List<Location> inputLocations = new ArrayList<>(inputKeys.size()); Map<String, PartitionKey> pathToKey = new HashMap<>(inputKeys.size()); for (PartitionKey key : inputKeys) { PartitionDetail partition = getPartition(key); String path = Objects.requireNonNull(partition).getRelativePath(); Location partitionLocation = files.getLocation(path); inputLocations.add(partitionLocation); pathToKey.put(partitionLocation.toURI().toString(), key); } Map<String, String> inputFormatConfiguration = files.getInputFormatConfiguration(inputLocations); inputFormatConfiguration.put(PATH_TO_PARTITIONING_MAPPING, GSON.toJson(pathToKey)); return inputFormatConfiguration; }
@Nullable @Override public TimePartitionDetail getPartitionByTime(long time) { PartitionDetail partitionDetail = getPartition(partitionKeyForTime(time)); return partitionDetail == null ? null : new BasicTimePartitionDetail(this, partitionDetail.getRelativePath(), partitionDetail.getPartitionKey(), partitionDetail.getMetadata()); }
@Nullable @Override public TimePartitionDetail getPartitionByTime(long time) { PartitionDetail partitionDetail = getPartition(partitionKeyForTime(time)); return partitionDetail == null ? null : new BasicTimePartitionDetail(this, partitionDetail.getRelativePath(), partitionDetail.getPartitionKey(), partitionDetail.getMetadata()); }
private void validateTimePartition(TimePartitionedFileSet dataset, long time, String path) { PartitionDetail partitionDetail = dataset.getPartitionByTime(time); Assert.assertEquals(path == null, partitionDetail == null); Assert.assertTrue(path == null || partitionDetail == null || path.equals(partitionDetail.getRelativePath())); }
@Override public void apply() throws Exception { PartitionDetail partitionDetail = dataset.getPartition(key); Assert.assertNotNull(partitionDetail); Assert.assertEquals(path, partitionDetail.getRelativePath()); } });
try { if (partitionLocation.exists()) { Location dstLocation = getQuarantineLocation().append(partition.getRelativePath()); Location dstParent = Locations.getParent(dstLocation); throw new DataSetException(String.format("Failed to move location %s into quarantine", partitionLocation), ioe); operationsInThisTx.add(new DropPartitionOperation(key, partition.getRelativePath()));
try { if (partitionLocation.exists()) { Location dstLocation = getQuarantineLocation().append(partition.getRelativePath()); Location dstParent = Locations.getParent(dstLocation); throw new DataSetException(String.format("Failed to move location %s into quarantine", partitionLocation), ioe); operationsInThisTx.add(new DropPartitionOperation(key, partition.getRelativePath()));
@GET @Path("partitions/{partition}/subpartitions/{sub-partition}") public void read(HttpServiceRequest request, HttpServiceResponder responder, @PathParam("partition") String partition, @PathParam("sub-partition") int subPartition) { PartitionDetail partitionDetail = pfs.getPartition(PartitionKey.builder() .addStringField("partition", partition) .addIntField("sub-partition", subPartition) .build()); if (partitionDetail == null) { responder.sendString(404, "Partition not found.", Charsets.UTF_8); return; } try { responder.send(200, partitionDetail.getLocation().append("file"), "text/plain"); } catch (IOException e) { responder.sendError(400, String.format("Unable to read path '%s'", partitionDetail.getRelativePath())); } }