@Override public boolean apply(WorkUnitState workUnitState) { return workUnitState.getWorkingState().equals(WorkingState.SUCCESSFUL); } }
@Override public boolean apply(WorkUnitState input) { return null == input || !WorkingState.SUCCESSFUL.equals(input.getWorkingState()); } };
@Override public void publishData(Collection<? extends WorkUnitState> states) throws IOException { super.publishData(states); for (WorkUnitState state : states) { if (state.getWorkingState() == WorkingState.COMMITTED) { try { deleteFilesOnSource(state); } catch (Throwable t) { log.warn( String.format("Failed to delete one or more files on source in %s", state.getProp(CopySource.SERIALIZED_COPYABLE_FILE)), t); } } else { log.info(String.format("Not deleting files %s on source fileSystem as the workunit state is %s.", state.getProp(CopySource.SERIALIZED_COPYABLE_FILE), state.getWorkingState())); } } }
@VisibleForTesting public void preservePartitionParams(Collection<? extends WorkUnitState> states) { for (WorkUnitState wus : states) { if (wus.getWorkingState() != WorkingState.COMMITTED) { continue; } if (!wus.contains(COMPLETE_SOURCE_PARTITION_NAME)) { continue; } if (!wus.contains(COMPLETE_DEST_PARTITION_NAME)) { continue; } if (!(wus.contains(PARTITION_PARAMETERS_WHITELIST) || wus.contains(PARTITION_PARAMETERS_BLACKLIST))) { continue; } List<String> whitelist = COMMA_SPLITTER.splitToList(wus.getProp(PARTITION_PARAMETERS_WHITELIST, StringUtils.EMPTY)); List<String> blacklist = COMMA_SPLITTER.splitToList(wus.getProp(PARTITION_PARAMETERS_BLACKLIST, StringUtils.EMPTY)); String completeSourcePartitionName = wus.getProp(COMPLETE_SOURCE_PARTITION_NAME); String completeDestPartitionName = wus.getProp(COMPLETE_DEST_PARTITION_NAME); if (!copyPartitionParams(completeSourcePartitionName, completeDestPartitionName, whitelist, blacklist)) { log.warn("Unable to copy partition parameters from " + completeSourcePartitionName + " to " + completeDestPartitionName); } } }
if (wus.getWorkingState() == WorkingState.SUCCESSFUL) { wus.setWorkingState(WorkUnitState.WorkingState.COMMITTED); if (copyEntity instanceof CopyableFile) { CopyableFile copyableFile = (CopyableFile) copyEntity; if (wus.getWorkingState() == WorkingState.COMMITTED) { CopyEventSubmitterHelper.submitSuccessfulFilePublish(this.eventSubmitter, copyableFile, wus);
private int persistFailedFileSet(Collection<? extends WorkUnitState> workUnitStates) throws IOException { int filesPersisted = 0; for (WorkUnitState wu : workUnitStates) { if (wu.getWorkingState() == WorkingState.SUCCESSFUL) { CopyEntity entity = CopySource.deserializeCopyEntity(wu); if (entity instanceof CopyableFile) { CopyableFile file = (CopyableFile) entity; Path outputDir = FileAwareInputStreamDataWriter.getOutputDir(wu); CopyableDatasetMetadata metadata = CopySource.deserializeCopyableDataset(wu); Path outputPath = FileAwareInputStreamDataWriter.getOutputFilePath(file, outputDir, file.getDatasetAndPartition(metadata)); if (this.recoveryHelper.persistFile(wu, file, outputPath)) { filesPersisted++; } } } } return filesPersisted; }