private Input.InputFormatProviderInput createInput(Input.DatasetInput datasetInput) { String datasetName = datasetInput.getName(); Map<String, String> datasetArgs = datasetInput.getArguments(); // keep track of the original alias to set it on the created Input before returning it String originalAlias = datasetInput.getAlias(); Dataset dataset; if (datasetInput.getNamespace() == null) { dataset = getDataset(datasetName, datasetArgs, AccessType.READ); } else { dataset = getDataset(datasetInput.getNamespace(), datasetName, datasetArgs, AccessType.READ); } DatasetInputFormatProvider datasetInputFormatProvider = new DatasetInputFormatProvider(datasetInput.getNamespace(), datasetName, datasetArgs, dataset, datasetInput.getSplits(), MapReduceBatchReadableInputFormat.class); return (Input.InputFormatProviderInput) Input.of(datasetName, datasetInputFormatProvider).alias(originalAlias); }
public void addInput(String stageName, Input input) { if (input instanceof Input.DatasetInput) { // Note if input format provider is trackable then it comes in as DatasetInput Input.DatasetInput datasetInput = (Input.DatasetInput) input; addInput(stageName, datasetInput.getName(), datasetInput.getAlias(), datasetInput.getArguments(), datasetInput.getSplits()); } else if (input instanceof Input.InputFormatProviderInput) { Input.InputFormatProviderInput ifpInput = (Input.InputFormatProviderInput) input; addInput(stageName, ifpInput.getAlias(), new BasicInputFormatProvider(ifpInput.getInputFormatProvider().getInputFormatClassName(), ifpInput.getInputFormatProvider().getInputFormatConfiguration())); } else if (input instanceof Input.StreamInput) { Input.StreamInput streamInput = (Input.StreamInput) input; addInput(stageName, streamInput.getAlias(), streamInput); } }
private Input.InputFormatProviderInput createInput(Input.DatasetInput datasetInput) { String datasetName = datasetInput.getName(); Map<String, String> datasetArgs = datasetInput.getArguments(); // keep track of the original alias to set it on the created Input before returning it String originalAlias = datasetInput.getAlias(); Dataset dataset; if (datasetInput.getNamespace() == null) { dataset = getDataset(datasetName, datasetArgs, AccessType.READ); } else { dataset = getDataset(datasetInput.getNamespace(), datasetName, datasetArgs, AccessType.READ); } DatasetInputFormatProvider datasetInputFormatProvider = new DatasetInputFormatProvider(datasetInput.getNamespace(), datasetName, datasetArgs, dataset, datasetInput.getSplits(), MapReduceBatchReadableInputFormat.class); return (Input.InputFormatProviderInput) Input.of(datasetName, datasetInputFormatProvider).alias(originalAlias); }