@Override public String getOutputFormatClassName() { return delegate.getOutputFormatClassName(); }
@Override public String getOutputFormatClassName() { checkNotExternal(); PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { return "co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputFormat"; } return files.getOutputFormatClassName(); }
@Override public String getOutputFormatClassName() { checkNotExternal(); PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { return "co.cask.cdap.internal.app.runtime.batch.dataset.partitioned.DynamicPartitioningOutputFormat"; } return files.getOutputFormatClassName(); }
@Override public Map<String, String> getOutputFormatConfiguration() { checkNotExternal(); // copy the output properties of the embedded file set to the output arguments Map<String, String> outputArgs = new HashMap<>(files.getOutputFormatConfiguration()); // we set the file set's output path in the definition's getDataset(), so there is no need to configure it again. // here we just want to validate that an output partition key or dynamic partitioner was specified in the arguments. PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(runtimeArguments); if (dynamicPartitionerClassName == null) { throw new DataSetException( "Either a Partition key or a DynamicPartitioner class must be given as a runtime argument."); } copyDynamicPartitionerArguments(runtimeArguments, outputArgs); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, files.getOutputFormatClassName()); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET, getName()); } else { assertNotExists(outputKey, true); } return ImmutableMap.copyOf(outputArgs); }
@Override public Map<String, String> getOutputFormatConfiguration() { checkNotExternal(); // copy the output properties of the embedded file set to the output arguments Map<String, String> outputArgs = new HashMap<>(files.getOutputFormatConfiguration()); // we set the file set's output path in the definition's getDataset(), so there is no need to configure it again. // here we just want to validate that an output partition key or dynamic partitioner was specified in the arguments. PartitionKey outputKey = PartitionedFileSetArguments.getOutputPartitionKey(runtimeArguments, getPartitioning()); if (outputKey == null) { String dynamicPartitionerClassName = PartitionedFileSetArguments.getDynamicPartitioner(runtimeArguments); if (dynamicPartitionerClassName == null) { throw new DataSetException( "Either a Partition key or a DynamicPartitioner class must be given as a runtime argument."); } copyDynamicPartitionerArguments(runtimeArguments, outputArgs); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_FORMAT_CLASS_NAME, files.getOutputFormatClassName()); outputArgs.put(Constants.Dataset.Partitioned.HCONF_ATTR_OUTPUT_DATASET, getName()); } else { assertNotExists(outputKey, true); } return ImmutableMap.copyOf(outputArgs); }
@Test public void testInputOutputFormatClassAtRuntime() throws Exception { // create a dataset with text input and output formats DatasetId datasetId = OTHER_NAMESPACE.dataset("testRuntimeFormats"); dsFrameworkUtil.createInstance("fileSet", datasetId, FileSetProperties.builder() .setInputFormat(TextInputFormat.class) .setOutputFormat(TextOutputFormat.class) .build()); // without passing anything in arguments, the input/output format classes will come from dataset properties FileSet fs = dsFrameworkUtil.getInstance(datasetId); Assert.assertEquals(TextInputFormat.class.getName(), fs.getInputFormatClassName()); Assert.assertEquals(TextOutputFormat.class.getName(), fs.getOutputFormatClassName()); // allow overriding the input format in dataset runtime args fs = dsFrameworkUtil.getInstance(datasetId, ImmutableMap.of( FileSetProperties.INPUT_FORMAT, CombineTextInputFormat.class.getName())); Assert.assertEquals(CombineTextInputFormat.class.getName(), fs.getInputFormatClassName()); Assert.assertEquals(TextOutputFormat.class.getName(), fs.getOutputFormatClassName()); // allow overriding both the input and output format in dataset runtime args fs = dsFrameworkUtil.getInstance(datasetId, ImmutableMap.of( FileSetProperties.INPUT_FORMAT, CombineTextInputFormat.class.getName(), FileSetProperties.OUTPUT_FORMAT, NullOutputFormat.class.getName())); Assert.assertEquals(CombineTextInputFormat.class.getName(), fs.getInputFormatClassName()); Assert.assertEquals(NullOutputFormat.class.getName(), fs.getOutputFormatClassName()); }