for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) { if (edgeEntry.getValue().getDataMovementType() == DataMovementType.CUSTOM && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName()
for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) { if (edgeEntry.getValue().getDataMovementType() == DataMovementType.CUSTOM && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName()
@Override public void initialize() { // this will prevent vertex from starting until we notify we are done getContext().vertexReconfigurationPlanned(); Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties(); // wait for sources and self to start numConfiguredSources = 0; configured = new AtomicBoolean(false); started = new AtomicBoolean(false); for (String entry : edges.keySet()) { getContext().registerForVertexStateUpdates(entry, EnumSet.of(VertexState.CONFIGURED)); } }
@Override public void initialize() { // this will prevent vertex from starting until we notify we are done getContext().vertexReconfigurationPlanned(); parallelismSet = false; numConfiguredSources = 0; configured = false; started = false; numSources = getContext().getInputVertexEdgeProperties().size(); // wait for sources and self to start Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties(); for (String entry : edges.keySet()) { getContext().registerForVertexStateUpdates(entry, EnumSet.of(VertexState.CONFIGURED)); } }
@Override public synchronized void onVertexStateUpdated(VertexStateUpdate stateUpdate) throws Exception { numConfiguredSources++; int target = getContext().getInputVertexEdgeProperties().size(); LOG.info("For vertex: " + getContext().getVertexName() + " Received configured signal from: " + stateUpdate.getVertexName() + " numConfiguredSources: " + numConfiguredSources + " needed: " + target); Preconditions.checkState(numConfiguredSources <= target, "Vertex: " + getContext().getVertexName()); if (numConfiguredSources == target) { configure(); } }
ByteString.copyFrom(getContext().getUserPayload().getPayload())); Map<String, EdgeProperty> edgePropertyMap = getContext().getInputVertexEdgeProperties(); Set<String> sourceVerticesDAG = edgePropertyMap.keySet(); Set<String> sourceVerticesConfig = new HashSet<>(config.getSourcesList());
for (Map.Entry<String, EdgeProperty> e : getContext().getInputVertexEdgeProperties().entrySet()) { if (e.getValue().getDataMovementType() == CUSTOM && e.getValue().getEdgeManagerDescriptor().getClassName()
@Override public void onVertexStarted(List<TaskAttemptIdentifier> completions) { managedTasks = getContext().getVertexNumTasks(getContext().getVertexName()); Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties(); for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) { String srcVertex = entry.getKey(); //track vertices with task count > 0 if (getContext().getVertexNumTasks(srcVertex) > 0) { LOG.info("Task count in " + srcVertex + ": " + getContext().getVertexNumTasks(srcVertex)); srcVertexConfigured.put(srcVertex, false); getContext().registerForVertexStateUpdates(srcVertex, EnumSet.of(VertexState.CONFIGURED)); } else { LOG.info("Vertex: " + getContext().getVertexName() + "; Ignoring " + srcVertex + " as it has got 0 tasks"); } } onVertexStartedDone.set(true); scheduleTasks(); }
+ " from " + currentParallelism); Map<String, EdgeProperty> edgeManagers = new HashMap<String, EdgeProperty>(); for(Map.Entry<String,EdgeProperty> entry : getContext().getInputVertexEdgeProperties().entrySet()) { EdgeProperty edge = entry.getValue(); edge = EdgeProperty.create(DataMovementType.SCATTER_GATHER, edge.getDataSourceType(), edge.getSchedulingType(),
@Override public synchronized void onVertexStarted(List<TaskAttemptIdentifier> completions) { Map<String, EdgeProperty> inputs = getContext().getInputVertexEdgeProperties(); for(Map.Entry<String, EdgeProperty> entry : inputs.entrySet()) { srcVertexInfo.put(entry.getKey(), createSourceVertexInfo(entry.getValue(),
@Override public void onVertexStarted(List<TaskAttemptIdentifier> completions) { Map<String, EdgeProperty> edges = getContext(). getInputVertexEdgeProperties(); for (Map.Entry<String, EdgeProperty> entry : edges.entrySet()) { String srcVertex = entry.getKey(); //track vertices with task count > 0 int numTasks = getContext().getVertexNumTasks(srcVertex); if (numTasks > 0) { LOG.info("Task count in " + srcVertex + ": " + numTasks); srcVertexInfo.put(srcVertex, createSourceVertexInfo(entry.getValue(), getContext().getVertexNumTasks(getContext().getVertexName()))); getContext().registerForVertexStateUpdates(srcVertex, EnumSet.of(VertexState.CONFIGURED)); } else { LOG.info("Vertex: " + getContext().getVertexName() + "; Ignoring " + srcVertex + " as it has " + numTasks + " tasks"); } } if (completions != null) { for (TaskAttemptIdentifier attempt : completions) { onSourceTaskCompleted(attempt); } } onVertexStartedDone.set(true); // track the tasks in this vertex updatePendingTasks(); processPendingTasks(); }
for (Entry<String, EdgeProperty> edgeEntry : context.getInputVertexEdgeProperties().entrySet()) { if (edgeEntry.getValue().getDataMovementType() == DataMovementType.CUSTOM && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName()
sourceTaskCompleted.put(sourceVertex, new BitSet()); for (String vertex : getContext().getInputVertexEdgeProperties().keySet()) { if (sourceVertices.indexOf(vertex) != -1) { getContext().registerForVertexStateUpdates(vertex, EnumSet.of(VertexState.CONFIGURED));
builder.addAllNumChunks(Ints.asList(this.numChunksPerSrc)); Map<String, EdgeProperty> edgeProperties = getContext().getInputVertexEdgeProperties(); Iterator<Map.Entry<String, EdgeProperty>> iter = edgeProperties.entrySet().iterator(); while (iter.hasNext()) {
Map<String, EdgeProperty> edges = getContext().getInputVertexEdgeProperties(); int oneToOneSrcTaskCount = 0; numOneToOneEdges = 0;
for(Map.Entry<String,EdgeProperty> entry : getContext().getInputVertexEdgeProperties().entrySet()) { EdgeProperty edge = entry.getValue(); edge = EdgeProperty.create(DataMovementType.SCATTER_GATHER, edge.getDataSourceType(), edge.getSchedulingType(),