inputDescriptor.setUserPayload(UserPayload.create(updatedPayload.toByteString().asReadOnlyByteBuffer())); } catch (IOException e) { e.printStackTrace();
inputDescriptor.setUserPayload(UserPayload.create(updatedPayload.toByteString().asReadOnlyByteBuffer())); } catch (IOException e) { e.printStackTrace();
InputInitializerDescriptor descriptor = InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()); InputDescriptor inputDescriptor = InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload .create(MRRuntimeProtos.MRInputUserPayloadProto.newBuilder() .setConfigurationBytes(TezUtils.createByteStringFromConf(conf))
input1Bb.putInt(0, 300); UserPayload input1Payload = UserPayload.create(input1Bb); InputDescriptor id1 = InputDescriptor.create("input1ClassName").setUserPayload(input1Payload); InputSpec inputSpec1 = new InputSpec("sourceVertexName1", id1, 33); InputSpec inputSpec2 = new InputSpec("sourceVertexName2", id1, 44);
private GroupInputEdge newGroupInputEdge(TezOperator fromOp, TezOperator toOp, VertexGroup from, Vertex to) throws IOException { EdgeProperty edgeProperty = newEdge(fromOp, toOp, true); String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName(); // In case of SCATTER_GATHER and UnorderedKVInput it will still be // ConcatenatedMergedKeyValueInput if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER) && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) { groupInputClass = OrderedGroupedMergedKVInput.class.getName(); } return GroupInputEdge.create(from, to, edgeProperty, InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())); }
@Private private static DataSourceDescriptor configureMRInputWithLegacySplitsGenerated(Configuration conf, boolean useLegacyInput) { InputDescriptor inputDescriptor; try { inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class .getName() : MRInput.class.getName()) .setUserPayload(MRInputHelpersInternal.createMRInputPayload(conf, null)); } catch (IOException e) { throw new TezUncheckedException(e); } DataSourceDescriptor dsd = DataSourceDescriptor.create(inputDescriptor, null, null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { dsd.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } return dsd; }
InputDescriptor inputDescriptor = InputDescriptor.create( inputClassName ).setUserPayload( edgeProperty.getEdgeDestination().getUserPayload() );
/** * This is a convenience method for creating an Edge descriptor based on the specified * EdgeManagerDescriptor. * * @param edgeManagerDescriptor the custom edge specification * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) { Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null"); EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for creating an Edge descriptor based on the specified * EdgeManagerDescriptor. * * @param edgeManagerDescriptor the custom edge specification * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) { Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null"); EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
private DataSourceDescriptor createGeneratorDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }
/** * This is a convenience method for creating an Edge descriptor based on the specified * EdgeManagerDescriptor. * * @param edgeManagerDescriptor the custom edge specification * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) { Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null"); EdgeProperty edgeProperty = EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
private DataSourceDescriptor createCustomDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor .create(InputDescriptor.create(inputClassName).setUserPayload(payload), customInitializerDescriptor, null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }
inputDescriptor.setUserPayload(UserPayload.create( ((InputUpdatePayloadEvent) event).getUserPayload())); } else if (event instanceof InputDataInformationEvent) {
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * * In this case - DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * If custom edge properties are required, the methods to get the relevant payloads should be * used. </p> * * In this case - DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultBroadcastEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * If custom edge properties are required, the methods to get the relevant payloads should be * used. </p> * * In this case - DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
/** * This is a convenience method for the typical usage of this edge, and creates an instance of * {@link org.apache.tez.dag.api.EdgeProperty} which is likely to be used. </p> * If custom edge properties are required, the methods to get the relevant payloads should be * used. </p> * * In this case - DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, * EdgeProperty.SchedulingType.SEQUENTIAL * * @return an {@link org.apache.tez.dag.api.EdgeProperty} instance */ public EdgeProperty createDefaultOneToOneEdgeProperty() { EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE, EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL, OutputDescriptor.create( getOutputClassName()).setUserPayload(getOutputPayload()), InputDescriptor.create( getInputClassName()).setUserPayload(getInputPayload())); Utils.setEdgePropertyHistoryText(this, edgeProperty); return edgeProperty; }
private DataSourceDescriptor createDistributorDataSource() throws IOException { InputSplitInfo inputSplitInfo; setupBasicConf(conf); try { inputSplitInfo = MRInputHelpers.generateInputSplitsToMem(conf, false, true, 0); } catch (Exception e) { throw new TezUncheckedException(e); } MRHelpers.translateMRConfToTez(conf); UserPayload payload = MRInputHelpersInternal.createMRInputPayload(conf, inputSplitInfo.getSplitsProto()); Credentials credentials = null; if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) { credentials = inputSplitInfo.getCredentials(); } DataSourceDescriptor ds = DataSourceDescriptor.create( InputDescriptor.create(inputClassName).setUserPayload(payload), InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()), inputSplitInfo.getNumTasks(), credentials, VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } return ds; }
private EdgeProperty createEdgeProperty( EdgeValues edgeValues ) { TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() ); outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) ); outputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( outputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( outputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( outputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues ); TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() ); inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) ); inputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( inputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( inputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( inputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues ); return EdgeProperty.create( edgeValues.getMovementType(), edgeValues.getSourceType(), edgeValues.getSchedulingType(), OutputDescriptor.create( edgeValues.getOutputClassName() ).setUserPayload( outputPayload ), InputDescriptor.create( edgeValues.getInputClassName() ).setUserPayload( inputPayload ) ); }
private EdgeProperty createEdgeProperty( EdgeValues edgeValues ) { TezConfiguration outputConfig = new TezConfiguration( edgeValues.getConfig() ); outputConfig.set( "cascading.node.sink", FlowElements.id( edgeValues.getFlowElement() ) ); outputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( outputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( outputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( outputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload outputPayload = createIntermediatePayloadOutput( outputConfig, edgeValues ); TezConfiguration inputConfig = new TezConfiguration( edgeValues.getConfig() ); inputConfig.set( "cascading.node.source", FlowElements.id( edgeValues.getFlowElement() ) ); inputConfig.set( "cascading.node.ordinals", Util.join( edgeValues.getOrdinals(), "," ) ); addFields( inputConfig, "cascading.node.key.fields", edgeValues.getResolvedKeyFieldsMap() ); addFields( inputConfig, "cascading.node.sort.fields", edgeValues.getResolvedSortFieldsMap() ); addFields( inputConfig, "cascading.node.value.fields", edgeValues.getResolvedValueFieldsMap() ); UserPayload inputPayload = createIntermediatePayloadInput( inputConfig, edgeValues ); return EdgeProperty.create( edgeValues.getMovementType(), edgeValues.getSourceType(), edgeValues.getSchedulingType(), OutputDescriptor.create( edgeValues.getOutputClassName() ).setUserPayload( outputPayload ), InputDescriptor.create( edgeValues.getInputClassName() ).setUserPayload( inputPayload ) ); }