InputInitializerDescriptor descriptor = InputInitializerDescriptor.create( HiveSplitGenerator.class.getName()); dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true)
@Override public InputInitializer run() throws Exception { InputInitializer initializer = ReflectionUtils .createClazzInstance(input.getControllerDescriptor().getClassName(), new Class[]{InputInitializerContext.class}, new Object[]{context}); return initializer; } });
public static InputInitializerDescriptor create(String initializerClassName) { return new InputInitializerDescriptor(initializerClassName); } }
@Override public UserPayload getUserPayload() { return this.input.getControllerDescriptor().getUserPayload(); }
: rootInputDescriptors.values()) { if (input.getControllerDescriptor() != null && input.getControllerDescriptor().getClassName() != null) { if (inputsWithInitializers == null) { inputsWithInitializers = Sets.newHashSet(); LOG.info("Starting root input initializer for input: " + input.getName() + ", with class: [" + input.getControllerDescriptor().getClassName() + "]");
InputInitializerDescriptor descriptor = InputInitializerDescriptor.create( HiveSplitGenerator.class.getName()); dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true) InputInitializerDescriptor descriptor = InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()); InputDescriptor inputDescriptor = InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload
public static InputInitializerDescriptor convertInputInitializerDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); InputInitializerDescriptor iid = InputInitializerDescriptor.create(className); setUserPayload(iid, payload); return iid; }
private DataSourceDescriptor createGeneratorDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }
private DAG createDAG(TezConfiguration tezConf) throws IOException { InputDescriptor inputDescriptor = InputDescriptor.create(FakeInput.class.getName()); InputInitializerDescriptor inputInitializerDescriptor = InputInitializerDescriptor.create(FakeInputInitializer.class.getName()); DataSourceDescriptor dataSourceDescriptor = DataSourceDescriptor.create(inputDescriptor, inputInitializerDescriptor, null);
InputInitializerDescriptor descriptor = InputInitializerDescriptor.create( HiveSplitGenerator.class.getName()); dataSource = MRInputLegacy.createConfigBuilder(conf, inputFormatClass).groupSplits(true)
private DataSourceDescriptor createDistributorDataSource() throws IOException { InputSplitInfo inputSplitInfo; setupBasicConf(conf); try { inputSplitInfo = MRInputHelpers.generateInputSplitsToMem(conf, false, true, 0); } catch (Exception e) { throw new TezUncheckedException(e); } MRHelpers.translateMRConfToTez(conf); UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf, inputSplitInfo.getSplitsProto()); Credentials credentials = null; if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) { credentials = inputSplitInfo.getCredentials(); } DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()), inputSplitInfo.getNumTasks(), credentials, VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } return ds; }
DataSourceDescriptor.create(InputDescriptor.create(MRInput.class.getName()) .setUserPayload(UserPayload.create(userPayLoadBuilder.build().toByteString().asReadOnlyByteBuffer())), InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()), inputSplitInfo.getNumTasks(), dag.getCredentials(),