/** * Suffix the alias of {@link Output} so that aliases of outputs are unique. */ private Output suffixOutput(Output output) { String suffixedAlias = String.format("%s-%s", output.getAlias(), UUID.randomUUID()); return output.alias(suffixedAlias); }
public ProvidedOutput(Output originalOutput, OutputFormatProvider outputFormatProvider) { this.output = originalOutput; this.outputFormatProvider = outputFormatProvider; this.outputFormatClassName = outputFormatProvider.getOutputFormatClassName(); this.outputFormatConfiguration = outputFormatProvider.getOutputFormatConfiguration(); if (outputFormatClassName == null) { throw new IllegalArgumentException(String.format("Output '%s' provided null as the output format", output.getAlias())); } if (outputFormatConfiguration == null) { throw new IllegalArgumentException(String.format("Output '%s' provided null as the output format configuration", output.getAlias())); } }
public ProvidedOutput(Output originalOutput, OutputFormatProvider outputFormatProvider) { this.output = originalOutput; this.outputFormatProvider = outputFormatProvider; this.outputFormatClassName = outputFormatProvider.getOutputFormatClassName(); this.outputFormatConfiguration = outputFormatProvider.getOutputFormatConfiguration(); if (outputFormatClassName == null) { throw new IllegalArgumentException(String.format("Output '%s' provided null as the output format", output.getAlias())); } if (outputFormatConfiguration == null) { throw new IllegalArgumentException(String.format("Output '%s' provided null as the output format configuration", output.getAlias())); } }
private Map<String, DatasetOutputCommitter> getDatasetOutputCommitters(List<ProvidedOutput> providedOutputs) { Map<String, DatasetOutputCommitter> datasetOutputCommitterOutputs = new HashMap<>(); for (ProvidedOutput providedOutput : providedOutputs) { if (providedOutput.getOutputFormatProvider() instanceof DatasetOutputCommitter) { datasetOutputCommitterOutputs.put(providedOutput.getOutput().getAlias(), (DatasetOutputCommitter) providedOutput.getOutputFormatProvider()); } } return datasetOutputCommitterOutputs; }
private Map<String, DatasetOutputCommitter> getDatasetOutputCommitters(List<ProvidedOutput> providedOutputs) { Map<String, DatasetOutputCommitter> datasetOutputCommitterOutputs = new HashMap<>(); for (ProvidedOutput providedOutput : providedOutputs) { if (providedOutput.getOutputFormatProvider() instanceof DatasetOutputCommitter) { datasetOutputCommitterOutputs.put(providedOutput.getOutput().getAlias(), (DatasetOutputCommitter) providedOutput.getOutputFormatProvider()); } } return datasetOutputCommitterOutputs; }
String umask = entry.getOutputFormatConfiguration().get(HADOOP_UMASK_PROPERTY); if (umask != null) { outputsWithUmask.add(entry.getOutput().getAlias()); outputUmasks.add(umask); for (int i = 0; i < outputs.size(); i++) { ProvidedOutput output = outputs.get(i); if (outputsWithUmask.contains(output.getOutput().getAlias())) { Map<String, String> outputConfig = new HashMap<>(output.getOutputFormatConfiguration()); outputConfig.remove(HADOOP_UMASK_PROPERTY);
String umask = entry.getOutputFormatConfiguration().get(HADOOP_UMASK_PROPERTY); if (umask != null) { outputsWithUmask.add(entry.getOutput().getAlias()); outputUmasks.add(umask); for (int i = 0; i < outputs.size(); i++) { ProvidedOutput output = outputs.get(i); if (outputsWithUmask.contains(output.getOutput().getAlias())) { Map<String, String> outputConfig = new HashMap<>(output.getOutputFormatConfiguration()); outputConfig.remove(HADOOP_UMASK_PROPERTY);
/** * Sets output formats and corresponding properties. * * @param job hadoop job on which configurations will be set * @param outputsMap list of outputs */ public static void setOutputs(Job job, List<ProvidedOutput> outputsMap) throws ClassNotFoundException { OutputFormatProvider rootOutputFormatProvider; rootOutputFormatProvider = getRootOutputFormatProvider(job, outputsMap); // Set root outputformat and its configuration for the Hadoop job. job.getConfiguration().set(ROOT_OUTPUT_FORMAT, rootOutputFormatProvider.getOutputFormatClassName()); for (Map.Entry<String, String> confEntry : rootOutputFormatProvider.getOutputFormatConfiguration().entrySet()) { job.getConfiguration().set(confEntry.getKey(), confEntry.getValue()); } for (ProvidedOutput output : outputsMap) { MultipleOutputs.addNamedOutput(job, output.getOutput().getAlias(), output.getOutputFormatClassName(), job.getOutputKeyClass(), job.getOutputValueClass(), output.getOutputFormatConfiguration()); } }
/** * Sets output formats and corresponding properties. * * @param job hadoop job on which configurations will be set * @param outputsMap list of outputs */ public static void setOutputs(Job job, List<ProvidedOutput> outputsMap) throws ClassNotFoundException { OutputFormatProvider rootOutputFormatProvider; rootOutputFormatProvider = getRootOutputFormatProvider(job, outputsMap); // Set root outputformat and its configuration for the Hadoop job. job.getConfiguration().set(ROOT_OUTPUT_FORMAT, rootOutputFormatProvider.getOutputFormatClassName()); for (Map.Entry<String, String> confEntry : rootOutputFormatProvider.getOutputFormatConfiguration().entrySet()) { job.getConfiguration().set(confEntry.getKey(), confEntry.getValue()); } for (ProvidedOutput output : outputsMap) { MultipleOutputs.addNamedOutput(job, output.getOutput().getAlias(), output.getOutputFormatClassName(), job.getOutputKeyClass(), job.getOutputValueClass(), output.getOutputFormatConfiguration()); } }
output.getName(), getProgram().getNamespaceId())); String alias = output.getAlias(); if (this.outputs.containsKey(alias)) { throw new IllegalArgumentException("Output already configured: " + alias);
output.getName(), getProgram().getNamespaceId())); String alias = output.getAlias(); if (this.outputs.containsKey(alias)) { throw new IllegalArgumentException("Output already configured: " + alias);