/** * Get a {@link DatasetAndPartition} instance for the dataset and fileSet this {@link CopyEntity} belongs to. * @param metadata {@link CopyableDatasetMetadata} for the dataset this {@link CopyEntity} belongs to. * @return an instance of {@link DatasetAndPartition} */ public DatasetAndPartition getDatasetAndPartition(CopyableDatasetMetadata metadata) { return new DatasetAndPartition(metadata, getFileSet()); }
private static Iterator<FileSet<CopyEntity>> partitionCopyableFiles(Dataset dataset, Collection<? extends CopyEntity> files) { Map<String, FileSet.Builder<CopyEntity>> partitionBuildersMaps = Maps.newHashMap(); for (CopyEntity file : files) { if (!partitionBuildersMaps.containsKey(file.getFileSet())) { partitionBuildersMaps.put(file.getFileSet(), new FileSet.Builder<>(file.getFileSet(), dataset)); } partitionBuildersMaps.get(file.getFileSet()).add(file); } return Iterators.transform(partitionBuildersMaps.values().iterator(), new Function<FileSet.Builder<CopyEntity>, FileSet<CopyEntity>>() { @Nullable @Override public FileSet<CopyEntity> apply(@Nonnull FileSet.Builder<CopyEntity> input) { return input.build(); } }); } }
workUnit.setProp(ConfigurationKeys.DATASET_URN_KEY, datasetAndPartition.toString()); workUnit.setProp(SlaEventKeys.DATASET_URN_KEY, this.copyableDataset.datasetURN()); workUnit.setProp(SlaEventKeys.PARTITION_KEY, copyEntity.getFileSet()); setWorkUnitWeight(workUnit, copyEntity, minWorkUnitWeight); setWorkUnitWatermark(workUnit, watermarkGenerator, copyEntity);
/** * Get a {@link DatasetAndPartition} instance for the dataset and fileSet this {@link CopyEntity} belongs to. * @param metadata {@link CopyableDatasetMetadata} for the dataset this {@link CopyEntity} belongs to. * @return an instance of {@link DatasetAndPartition} */ public DatasetAndPartition getDatasetAndPartition(CopyableDatasetMetadata metadata) { return new DatasetAndPartition(metadata, getFileSet()); }
private static Iterator<FileSet<CopyEntity>> partitionCopyableFiles(Dataset dataset, Collection<? extends CopyEntity> files) { Map<String, FileSet.Builder<CopyEntity>> partitionBuildersMaps = Maps.newHashMap(); for (CopyEntity file : files) { if (!partitionBuildersMaps.containsKey(file.getFileSet())) { partitionBuildersMaps.put(file.getFileSet(), new FileSet.Builder<>(file.getFileSet(), dataset)); } partitionBuildersMaps.get(file.getFileSet()).add(file); } return Iterators.transform(partitionBuildersMaps.values().iterator(), new Function<FileSet.Builder<CopyEntity>, FileSet<CopyEntity>>() { @Nullable @Override public FileSet<CopyEntity> apply(@Nonnull FileSet.Builder<CopyEntity> input) { return input.build(); } }); } }
workUnit.setProp(ConfigurationKeys.DATASET_URN_KEY, datasetAndPartition.toString()); workUnit.setProp(SlaEventKeys.DATASET_URN_KEY, this.copyableDataset.datasetURN()); workUnit.setProp(SlaEventKeys.PARTITION_KEY, copyEntity.getFileSet()); setWorkUnitWeight(workUnit, copyEntity, minWorkUnitWeight); setWorkUnitWatermark(workUnit, watermarkGenerator, copyEntity);