return GroupInputEdge.create(group, w, createEdgeProperty(w, edgeProp, vConf, work, tezWork), InputDescriptor.create(mergeInputClass.getName()));
if (!vertexGroups.contains(edge.getInputVertexGroup())) { throw new IllegalArgumentException( "Input vertex " + edge.getInputVertexGroup() + " doesn't exist!"); if (!vertices.containsValue(edge.getOutputVertex())) { throw new IllegalArgumentException( "Output vertex " + edge.getOutputVertex() + " doesn't exist!"); VertexGroup av = edge.getInputVertexGroup(); av.addOutputVertex(edge.getOutputVertex(), edge); groupInputEdges.add(edge); Vertex dstVertex = edge.getOutputVertex(); VertexGroup uv = edge.getInputVertexGroup(); for (Vertex member : uv.getMembers()) { newEdges.add(Edge.create(member, dstVertex, edge.getEdgeProperty()));
/** * An Edge that connects a {@link VertexGroup} to a destination Vertex. The * framework takes care of connecting the {@link VertexGroup} members with the * destination vertex. The tasks of the destination vertex see only 1 input * named after the VertexGroup instead of individual inputs from group * members. These individual inputs are merged using the mergedInput before * presenting them to the destination task. * * @param inputVertexGroup * source {@link VertexGroup} * @param outputVertex * destination Vertex * @param edgeProperty * the common {@link EdgeProperty} for this {@link GroupInputEdge} * @param mergedInput * {@link MergedLogicalInput} This input is responsible for merging * the data from the input vertex tasks to create a single input for * the output vertex tasks */ public static GroupInputEdge create(VertexGroup inputVertexGroup, Vertex outputVertex, EdgeProperty edgeProperty, InputDescriptor mergedInput) { return new GroupInputEdge(inputVertexGroup, outputVertex, edgeProperty, mergedInput); }
void addOutputVertex(Vertex outputVertex, GroupInputEdge edge) { this.groupInfo.edgeMergedInputs.put(outputVertex.getName(), edge.getMergedInput()); }
return GroupInputEdge.create(group, w, createEdgeProperty(edgeProp, vConf), InputDescriptor.create(mergeInputClass.getName()));
private GroupInputEdge newGroupInputEdge(TezOperator fromOp, TezOperator toOp, VertexGroup from, Vertex to) throws IOException { EdgeProperty edgeProperty = newEdge(fromOp, toOp, true); String groupInputClass = ConcatenatedMergedKeyValueInput.class.getName(); // In case of SCATTER_GATHER and UnorderedKVInput it will still be // ConcatenatedMergedKeyValueInput if(edgeProperty.getDataMovementType().equals(DataMovementType.SCATTER_GATHER) && edgeProperty.getEdgeDestination().getClassName().equals(OrderedGroupedKVInput.class.getName())) { groupInputClass = OrderedGroupedMergedKVInput.class.getName(); } return GroupInputEdge.create(from, to, edgeProperty, InputDescriptor.create(groupInputClass).setUserPayload(edgeProperty.getEdgeDestination().getUserPayload())); }
dag.addEdge( GroupInputEdge.create( vertexGroup, targetVertex, edgeProperty, inputDescriptor ) );
dag.addEdge( GroupInputEdge.create( vertexGroup, targetVertex, edgeProperty, inputDescriptor ) );
Edge.create(mapVertex3, checkerVertex, edgeConf.createDefaultEdgeProperty())) .addEdge( GroupInputEdge.create(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(), InputDescriptor.create( ConcatenatedMergedKeyValuesInput.class.getName())));
return GroupInputEdge.create(group, w, createEdgeProperty(edgeProp, vConf), InputDescriptor.create(mergeInputClass.getName()));
.setFromConfiguration(tezConf) .build(); GroupInputEdge edge = GroupInputEdge.create(uv12, v3, edgeConfig.createDefaultEdgeProperty(), InputDescriptor.create( ConcatenatedMergedKeyValuesInput.class.getName()));