public static Builder fromOriginAndDestination(FileSystem originFs, FileStatus origin, Path destination, CopyConfiguration copyConfiguration) { return _hiddenBuilder().originFS(originFs).origin(origin).destination(destination).configuration(copyConfiguration) .preserve(copyConfiguration.getPreserve()); }
@Override public Iterator<FileSet<CopyEntity>> getFileSetIterator(FileSystem targetFs, CopyConfiguration configuration) throws IOException { return partitionCopyableFiles(this.dataset, this.dataset.getCopyableFiles(targetFs, configuration)); }
/** * Get a {@link DatasetAndPartition} instance for the dataset and fileSet this {@link CopyEntity} belongs to. * @param metadata {@link CopyableDatasetMetadata} for the dataset this {@link CopyEntity} belongs to. * @return an instance of {@link DatasetAndPartition} */ public DatasetAndPartition getDatasetAndPartition(CopyableDatasetMetadata metadata) { return new DatasetAndPartition(metadata, getFileSet()); }
@VisibleForTesting protected CopyableFileGenerator getCopyableFileGenetator(FileSystem targetFs, CopyConfiguration configuration, TimestampedDatasetVersion copyableVersion, ConcurrentLinkedQueue<CopyableFile> copyableFileList) { return new CopyableFileGenerator(this.srcFs, targetFs, configuration, this.datasetRoot, this.getTargetRoot(configuration.getPublishDir()), copyableVersion.getDateTime(), copyableVersion.getPaths(), copyableFileList, this.copyableFileFilter()); }
@Test public void testBoundedAdd() throws IOException { ConcurrentBoundedWorkUnitList list = new ConcurrentBoundedWorkUnitList(10, new AllEqualComparator<FileSet<CopyEntity>>(), 1); Assert.assertTrue(addFiles(list, "fs", 6)); Assert.assertFalse(list.hasRejectedFileSet()); Assert.assertFalse(addFiles(list, "fs", 5)); Assert.assertTrue(list.hasRejectedFileSet()); Assert.assertTrue(addFiles(list, "fs", 4)); Assert.assertFalse(addFiles(list, "fs", 1)); }
/** * Read a {@link org.apache.gobblin.data.management.copy.OwnerAndPermission} from a {@link java.io.DataInput}. * @throws IOException */ public static OwnerAndPermission read(DataInput input) throws IOException { OwnerAndPermission oap = new OwnerAndPermission(); oap.readFields(input); return oap; } }
/** * Deserializes the serialized {@link List} of {@link CopyEntity} string. * Used together with {@link #serializeList(List)} * * @param serialized string * @return a new {@link List} of {@link CopyEntity}s */ public static List<CopyEntity> deserializeList(String serialized) { return GSON.fromJson(getSerializedWithNewPackage(serialized), new TypeToken<List<CopyEntity>>() {}.getType()); }
@Override public CopyableDataset datasetAtPath(Path path) throws IOException { return new TimeAwareRecursiveCopyableDataset(this.fs, path, this.props, this.datasetPattern); } }
@Override public String toString() { return serialize(this); }
public static CopyConfigurationBuilder builder(FileSystem targetFs, Properties properties) { return new CopyConfigurationBuilder(targetFs, properties); }
/** * Set number of {@link CopyEntity}s in {@link org.apache.gobblin.data.management.partition.FileSet}. */ public CopyResourceRequirementBuilder setEntities(int numberOfEntities) { setRequirement(ENTITIES_DIMENSION, (double) numberOfEntities); return this; }
/** * @return true if attribute should be preserved. */ public boolean preserve(Option option) { return 0 < (this.options & (1 << option.ordinal())); }
@Override public String toString() { return this.file.toString(); } }
@Override public CopyableDataset datasetAtPath(Path path) throws IOException { return new RecursiveCopyableDataset(this.fs, path, this.props, this.datasetPattern); } }
/** * Used for simulate runs. Should explain what this copy entity will do. */ public String explain() { return toString(); }
@Override public ResourceRequirement.Builder getResourceRequirementBuilder() { return getCopyResourceRequirementBuilder(); }
public CopyableDatasetMetadata(CopyableDatasetBase copyableDataset) { this.datasetURN = copyableDataset.datasetURN(); }
/** * Add a file set to the container. * @param fileSet File set, expressed as a {@link org.apache.gobblin.data.management.partition.FileSet} of {@link CopyEntity}s. * @param workUnits List of {@link WorkUnit}s corresponding to this file set. * @return true if the file set was added to the container, false otherwise (i.e. has reached max size). */ public boolean addFileSet(FileSet<CopyEntity> fileSet, List<WorkUnit> workUnits) { boolean addedWorkunits = addFileSetImpl(fileSet, workUnits); if (!addedWorkunits) { this.rejectedFileSet = true; } return addedWorkunits; }
@Override protected CopyableFileGenerator getCopyableFileGenetator(FileSystem targetFs, CopyConfiguration configuration, TimestampedDatasetVersion copyableVersion, ConcurrentLinkedQueue<CopyableFile> copyableFileList) { return new CopyableFileGeneratorForTest(copyableFileList, copyableVersion); }
/** * Set total bytes to copy in {@link org.apache.gobblin.data.management.partition.FileSet}. */ public CopyResourceRequirementBuilder setBytes(long totalBytes) { setRequirement(BYTES_DIMENSION, (double) totalBytes); return this; } }