private DataSourceDescriptor createGeneratorDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }
private DataSourceDescriptor createDistributorDataSource() throws IOException { InputSplitInfo inputSplitInfo; setupBasicConf(conf); try { inputSplitInfo = MRInputHelpers.generateInputSplitsToMem(conf, false, true, 0); } catch (Exception e) { throw new TezUncheckedException(e); } MRHelpers.translateMRConfToTez(conf); UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf, inputSplitInfo.getSplitsProto()); Credentials credentials = null; if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) { credentials = inputSplitInfo.getCredentials(); } DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()), inputSplitInfo.getNumTasks(), credentials, VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } return ds; }
private DataSourceDescriptor createCustomDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor .create(InputDescriptor.create(inputClassName).setUserPayload(payload), customInitializerDescriptor, null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }