InputDescriptor inputDescriptor = InputDescriptor.create(MRInputLegacy.class.getName()) .setUserPayload(UserPayload .create(MRRuntimeProtos.MRInputUserPayloadProto.newBuilder() .setConfigurationBytes(TezUtils.createByteStringFromConf(conf))
MRInputHelpers.parseMRInputPayload(inputDescriptor.getUserPayload()); this.conf = TezUtils.createConfFromByteString(protoPayload.getConfigurationBytes()); inputDescriptor.setUserPayload(UserPayload.create(updatedPayload.toByteString().asReadOnlyByteBuffer())); } catch (IOException e) { e.printStackTrace();
private void verifyInputSpecAndProto(InputSpec inputSpec, IOSpecProto inputSpecProto) { assertEquals(inputSpec.getPhysicalEdgeCount(), inputSpecProto.getPhysicalEdgeCount()); assertEquals(inputSpec.getSourceVertexName(), inputSpecProto.getConnectedVertexName()); assertEquals(inputSpec.getInputDescriptor().getClassName(), inputSpecProto.getIoDescriptor().getClassName()); assertEquals(inputSpec.getInputDescriptor().getUserPayload().getPayload(), inputSpecProto.getIoDescriptor().getUserPayload().getUserPayload().asReadOnlyByteBuffer()); }
@Private private static DataSourceDescriptor configureMRInputWithLegacySplitsGenerated(Configuration conf, boolean useLegacyInput) { InputDescriptor inputDescriptor; try { inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class .getName() : MRInput.class.getName()) .setUserPayload(MRInputHelpersInternal.createMRInputPayload(conf, null)); } catch (IOException e) { throw new TezUncheckedException(e); } DataSourceDescriptor dsd = DataSourceDescriptor.create(inputDescriptor, null, null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { dsd.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } return dsd; }
InputDescriptor inputDescriptor = InputDescriptor.create( inputClassName ).setUserPayload( edgeProperty.getEdgeDestination().getUserPayload() );
private GroupInputEdge newGroupInputEdge(TezOperator fromOp, TezOperator toOp, VertexGroup from, Vertex to) throws IOException { EdgeProperty edgeProperty = newEdge(fromOp, toOp, true); String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName(); // In case of SCATTER_GATHER and UnorderedKVInput it will still be // ConcatenatedMergedKeyValueInput if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER) && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) { groupInputClass = OrderedGroupedMergedKVInput.class.getName(); } return GroupInputEdge.create(from, to, edgeProperty, InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())); }
private static InputDescriptor convertInputDescriptorFromProto( EntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertPayloadFromProto(proto); InputDescriptor id = InputDescriptor.create(className); setUserPayload(id, payload); return id; }
private List<String> getSourceInterestList(List<InputSpec> inputSpecList) { List<String> sourcesOfInterest = Collections.emptyList(); if (inputSpecList != null) { boolean alreadyFound = false; for (InputSpec inputSpec : inputSpecList) { if (LlapTezUtils.isSourceOfInterest(inputSpec.getInputDescriptor().getClassName())) { if (!alreadyFound) { alreadyFound = true; sourcesOfInterest = new LinkedList<>(); } sourcesOfInterest.add(inputSpec.getSourceVertexName()); } } } return sourcesOfInterest; }
@Override public UserPayload getInputUserPayload() { return this.input.getIODescriptor().getUserPayload(); }
inputDescriptor.setUserPayload(UserPayload.create( ((InputUpdatePayloadEvent) event).getUserPayload())); } else if (event instanceof InputDataInformationEvent) {
public static Map<String,Object> convertEdgeProperty( EdgeProperty edge) { Map<String, Object> jsonDescriptor = new HashMap<String, Object>(); jsonDescriptor.put(DATA_MOVEMENT_TYPE_KEY, edge.getDataMovementType().name()); jsonDescriptor.put(DATA_SOURCE_TYPE_KEY, edge.getDataSourceType().name()); jsonDescriptor.put(SCHEDULING_TYPE_KEY, edge.getSchedulingType().name()); jsonDescriptor.put(EDGE_SOURCE_CLASS_KEY, edge.getEdgeSource().getClassName()); jsonDescriptor.put(EDGE_DESTINATION_CLASS_KEY, edge.getEdgeDestination().getClassName()); String history = edge.getEdgeSource().getHistoryText(); if (history != null) { jsonDescriptor.put(OUTPUT_USER_PAYLOAD_AS_TEXT, history); } history = edge.getEdgeDestination().getHistoryText(); if (history != null) { jsonDescriptor.put(INPUT_USER_PAYLOAD_AS_TEXT, history); } EdgeManagerPluginDescriptor descriptor = edge.getEdgeManagerDescriptor(); if (descriptor != null) { jsonDescriptor.put(EDGE_MANAGER_CLASS_KEY, descriptor.getClassName()); if (descriptor.getHistoryText() != null && !descriptor.getHistoryText().isEmpty()) { jsonDescriptor.put(USER_PAYLOAD_AS_TEXT, descriptor.getHistoryText()); } } return jsonDescriptor; }
@Override public void readFields(DataInput in) throws IOException { sourceVertexName = StringInterner.weakIntern(in.readUTF()); physicalEdgeCount = in.readInt(); inputDescriptor = new InputDescriptor(); inputDescriptor.readFields(in); }
/** * Modify the EdgeProperty to set the history text if available * @param edgeConfig Edge config * @param edgeProperty Edge property to be be modified */ static void setEdgePropertyHistoryText(HadoopKeyValuesBasedBaseEdgeConfig edgeConfig, EdgeProperty edgeProperty) { String inputHistoryText = edgeConfig.getInputHistoryText(); if (inputHistoryText != null) { edgeProperty.getEdgeDestination().setHistoryText(inputHistoryText); } String outputHistoryText = edgeConfig.getOutputHistoryText(); if (outputHistoryText != null) { edgeProperty.getEdgeSource().setHistoryText(outputHistoryText); } }
public static InputDescriptor create(String inputClassName) { return new InputDescriptor(inputClassName); } }
private DataSourceDescriptor createCustomDataSource() throws IOException { setupBasicConf(conf); MRHelpers.translateMRConfToTez(conf); Collection<URI> uris = maybeGetURIsForCredentials(); UserPayload payload = MRInputHelpersInternal.createMRInputPayload( conf, groupSplitsInAM, sortSplitsInAM); DataSourceDescriptor ds = DataSourceDescriptor .create(InputDescriptor.create(inputClassName).setUserPayload(payload), customInitializerDescriptor, null); if (conf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT, TezRuntimeConfiguration.TEZ_RUNTIME_CONVERT_USER_PAYLOAD_TO_HISTORY_TEXT_DEFAULT)) { ds.getInputDescriptor().setHistoryText(TezUtils.convertToHistoryText(conf)); } if (uris != null) { ds.addURIsForCredentials(uris); } return ds; }
InputDescriptor inputDescriptor = InputDescriptor.create( inputClassName ).setUserPayload( edgeProperty.getEdgeDestination().getUserPayload() );
InputDescriptor.create(mergeInputClass.getName()));
public String toString() { return "{ sourceVertexName=" + sourceVertexName + ", physicalEdgeCount=" + physicalEdgeCount + ", inputClassName=" + inputDescriptor.getClassName() + " }"; }
private void initializeGroupInputs() throws TezException { if (groupInputSpecs != null && !groupInputSpecs.isEmpty()) { groupInputsMap = new ConcurrentHashMap<String, MergedLogicalInput>(groupInputSpecs.size()); for (GroupInputSpec groupInputSpec : groupInputSpecs) { if (LOG.isDebugEnabled()) { LOG.debug("Initializing GroupInput using GroupInputSpec: " + groupInputSpec); } MergedInputContext mergedInputContext = new TezMergedInputContextImpl(groupInputSpec.getMergedInputDescriptor().getUserPayload(), groupInputSpec.getGroupName(), groupInputsMap, inputReadyTracker, localDirs, this); List<Input> inputs = Lists.newArrayListWithCapacity(groupInputSpec.getGroupVertices().size()); for (String groupVertex : groupInputSpec.getGroupVertices()) { inputs.add(inputsMap.get(groupVertex)); } MergedLogicalInput groupInput = (MergedLogicalInput) createMergedInput(groupInputSpec.getMergedInputDescriptor(), mergedInputContext, inputs); groupInputsMap.put(groupInputSpec.getGroupName(), groupInput); } } }
@Override public void readFields(DataInput in) throws IOException { groupName = StringInterner.weakIntern(Text.readString(in)); int numMembers = in.readInt(); groupVertices = Lists.newArrayListWithCapacity(numMembers); for (int i=0; i<numMembers; ++i) { groupVertices.add(StringInterner.weakIntern(Text.readString(in))); } mergedInputDescriptor = new InputDescriptor(); mergedInputDescriptor.readFields(in); }