InputInitializerDescriptor descriptor = InputInitializerDescriptor.create( HiveSplitGenerator.class.getName()); dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true)
InputInitializerDescriptor descriptor = InputInitializerDescriptor.create( HiveSplitGenerator.class.getName()); dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true) InputInitializerDescriptor descriptor = InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()); InputDescriptor inputDescriptor = InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload
public static InputInitializerDescriptor convertInputInitializerDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); InputInitializerDescriptor iid = InputInitializerDescriptor.create(className); setUserPayload(iid, payload); return iid; }
private DataSourceDescriptor createGeneratorDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }
private DAG createDAG(TezConfiguration tezConf) throws IOException { InputDescriptor inputDescriptor = InputDescriptor.create(FakeInput.class.getName()); InputInitializerDescriptor inputInitializerDescriptor = InputInitializerDescriptor.create(FakeInputInitializer.class.getName()); DataSourceDescriptor dataSourceDescriptor = DataSourceDescriptor.create(inputDescriptor, inputInitializerDescriptor, null);
InputInitializerDescriptor descriptor = InputInitializerDescriptor.create( HiveSplitGenerator.class.getName()); dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true)
private DataSourceDescriptor createDistributorDataSource() throws IOException { InputSplitInfo inputSplitInfo; setupBasicConf(conf); try { inputSplitInfo = MRInputHelpers.generateInputSplitsToMem(conf, false, true, 0); } catch (Exception e) { throw new TezUncheckedException(e); } MRHelpers.translateMRConfToTez(conf); UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf, inputSplitInfo.getSplitsProto()); Credentials credentials = null; if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) { credentials = inputSplitInfo.getCredentials(); } DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()), inputSplitInfo.getNumTasks(), credentials, VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } return ds; }
DataSourceDescriptor.create(InputDescriptor.create(MRInput.class.getName()) .setUserPayload(UserPayload.create(userPayLoadBuilder.build().toByteString().asReadOnlyByteBuffer())), InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()), inputSplitInfo.getNumTasks(), dag.getCredentials(),