@Override public int getConsumerId(int pipeId) { return GraphManager.getRingConsumerId(graphManager, pipeId); }});
public static int getRingConsumerStageId(GraphManager gm, int ringId) { return getRingConsumerId(gm, ringId); }
public static PronghornStage getRingConsumer(GraphManager gm, int ringId) { int stageId = getRingConsumerId(gm, ringId); if (stageId<0) { throw new UnsupportedOperationException("Can not find stage reading from pipe "+ringId+" Check graph construction."); } return gm.stageIdToStage[stageId]; }
private static float computeWeightBetweenStages(GraphManager m, int consumerStage, int producerStage) { float weight = 0f; Pipe[] pipes = m.pipeIdToPipe; int p = pipes.length; while (--p >= 0) { if (null != pipes[p]) { if (producerStage == GraphManager.getRingProducerId(m, pipes[p].id) && consumerStage == GraphManager.getRingConsumerId(m, pipes[p].id)) { weight += 2; } else if (consumerStage == GraphManager.getRingProducerId(m, pipes[p].id) && producerStage == GraphManager.getRingConsumerId(m, pipes[p].id)) { weight += 2; } } } return weight; }
private static boolean allTheSame(GraphManager m, int producer, int outputPipeCount) { boolean allTheSame = true; int lastConsumer = -1; for(int w=1; w<=outputPipeCount; w++) { int consumerId = GraphManager.getRingConsumerId(m, GraphManager.getOutputPipe(m, producer, w).id ); if (-1 == lastConsumer) { lastConsumer = consumerId; } else { if (consumerId != lastConsumer) { allTheSame = false; break; } } } return allTheSame; }
private static boolean isSameDestination(GraphManager m, int producer, int a, int b) { Pipe<?> outputPipe1 = GraphManager.getOutputPipe(m, producer, a); Pipe<?> outputPipe2 = GraphManager.getOutputPipe(m, producer, b); int con1 = GraphManager.getRingConsumerId(m, outputPipe1.id); int con2 = GraphManager.getRingConsumerId(m, outputPipe2.id); return (con1==con2 && Pipe.isForSameSchema(outputPipe1, outputPipe2) ); }
private static <T extends MessageSchema<T>> Pipe<T>[] pipesOfTypeWithNoConsumer(int count, int p, GraphManager gm, Class<T> targetSchemaClazz) { //pass one to count all the instances while (--p>=0) { Pipe tp = gm.pipeIdToPipe[p]; if ((null != tp) && Pipe.isForSchema(tp, targetSchemaClazz) && (getRingConsumerId(gm, tp.id)==-1)) { Pipe<T>[] result = pipesOfTypeWithNoConsumer(count+1,p,gm,targetSchemaClazz); result[(result.length-1)-count] = tp; return result; } } return (0==count) ? EMPTY_PIPE_ARRAY : new Pipe[count]; }
private boolean countParallelProducers(int consumerId, int producerId, GraphManager graphManager, Pipe p) { int noMatchCount = 0; int proOutCount = GraphManager.getOutputPipeCount(graphManager, producerId); while (--proOutCount>=0) { //all the other output coming from the producer Pipe outputPipe = GraphManager.getOutputPipe(graphManager, producerId, 1+proOutCount); //is this the same schema as the pipe in question. if (Pipe.isForSameSchema(outputPipe, p)) { //determine if they are consumed by the same place or not int conId = GraphManager.getRingConsumerId(graphManager, outputPipe.id); if (consumerId != conId) { //only count if they are not consumed at the same place noMatchCount++; } //else if they are consumed by the same place then the pipes are just for parallel storage not parallel compute } } return noMatchCount>=1; }
private static Pipe[] buildProducersPipes(int count, int indexesIdx, int outputIdx, final int[] indexes, final PronghornStage[] stages, final GraphManager graphManager) { while(indexesIdx<indexes.length) { int outputCount = GraphManager.getOutputPipeCount(graphManager, stages[indexes[indexesIdx]].stageId); while(outputIdx<=outputCount) { Pipe pipe = GraphManager.getOutputPipe(graphManager, stages[indexes[indexesIdx]], outputIdx); //is the consumer of this pipe inside the graph? int consumerId = GraphManager.getRingConsumerId(graphManager, pipe.id); int k = stages.length; while (--k>=0) { if (stages[k].stageId==consumerId) { Pipe[] result = buildProducersPipes(count+1, indexesIdx, outputIdx+1, indexes, stages, graphManager); result[count] = pipe; return result; } } outputIdx++; } outputIdx=1; indexesIdx++; } return new Pipe[count]; }
private static void visitDownstream(int stageId, GraphManager graphManager, long[] bitMap, int depth, GraphVisitor visitor) { assert(stageId<=graphManager.stageCounter.get()) : "This stage id is out of range"; if (stageId<0) { //nothing to visit return; } int mapIndex = stageId>>6;//divide by 64 int mapBit = 1<<(stageId&0x3F); if ((bitMap[mapIndex]&mapBit)!=0) { return;//we have encountered some place we have already been. } else { //mark this as a new place we have been bitMap[mapIndex] |= mapBit; //now call everyone downstream //visit children only if visitor returns true if (visitor.visit(graphManager, stageId, depth)) { int c = GraphManager.getOutputPipeCount(graphManager, stageId); for(int i = 1; i<=c; i++) { int outPipeId = GraphManager.getOutputPipe(graphManager, stageId, c).id; visitDownstream(GraphManager.getRingConsumerId(graphManager, outPipeId), graphManager, bitMap, depth+1, visitor); } } } }
private static Pipe[] buildProducersPipes(int count, int indexesIdx, int outputIdx, final int[] indexes, final PronghornStage[] stages, final GraphManager graphManager) { while (indexesIdx < indexes.length) { int outputCount = GraphManager.getOutputPipeCount(graphManager, stages[indexes[indexesIdx]].stageId); while (outputIdx <= outputCount) { Pipe pipe = GraphManager.getOutputPipe(graphManager, stages[indexes[indexesIdx]], outputIdx); //is the consumer of this pipe inside the graph? int consumerId = GraphManager.getRingConsumerId(graphManager, pipe.id); int k = stages.length; while (--k >= 0) { if (stages[k].stageId == consumerId) { Pipe[] result = buildProducersPipes(count + 1, indexesIdx, outputIdx + 1, indexes, stages, graphManager); result[count] = pipe; return result; } } outputIdx++; } outputIdx = 1; indexesIdx++; } return new Pipe[count]; }
private static void regInput(GraphManager gm, Pipe input, int stageId) { if (null!=input) { int inputId = input.id; Pipe.structRegistry(input,gm.recordTypeData); try { gm.ringIdToStages = setValue(gm.ringIdToStages, (inputId*2)+1, stageId, gm.stageIdToStage[stageId]); //source +0 then target +1 gm.pipeIdToPipe = setValue(gm.pipeIdToPipe, inputId, input); gm.multInputIds = setValue(gm.multInputIds, gm.topInput++, inputId, input); } catch (UnsupportedOperationException uoe) { input.creationStack(); logger.info("\nRegister: "+input); throw uoe; } assert(stageId == getRingConsumerId(gm, inputId)); } }
if (!useMoreThreads) { int conId = GraphManager.getRingConsumerId(graphManager, outputPipe.id); if ((totalOutputsCount-1)==proOutCount && conId == consumerId) {
private void add(PronghornStage[] pronghornStages, PronghornStage stage, final int root, final GraphManager graphManager, final IntHashTable rootsTable) { int i = 0; while (i<pronghornStages.length && pronghornStages[i]!=null) { if (pronghornStages[i]==stage) { return;//already added } i++; } //now add the new stage at index i pronghornStages[i]=stage; //Recursively add the ones under the same root. int outputCount = GraphManager.getOutputPipeCount(graphManager, stage.stageId); for(int r = 1; r<=outputCount; r++) { Pipe outputPipe = GraphManager.getOutputPipe(graphManager, stage, r); int consumerId = GraphManager.getRingConsumerId(graphManager, outputPipe.id); //this exists and has the same root so add it if (consumerId>=0 && rootId(consumerId, rootsTable)==root) { add(pronghornStages, GraphManager.getStage(graphManager, consumerId), root, graphManager, rootsTable); } } }
public static void enableBatching(GraphManager gm) { int j = gm.pipeIdToPipe.length; while (--j>=0) { Pipe ring = gm.pipeIdToPipe[j]; //never enable batching on the monitor rings if (null!=ring && !ringHoldsMonitorData(gm, ring) ) { int ringId1 = ring.id; int stageId1 = GraphManager.getRingConsumerId(gm, ringId1); if (stageId1>=0) { if (PronghornStage.supportsBatchedRelease(gm.stageIdToStage[stageId1])) { Pipe.setMaxReleaseBatchSize(ring); } } int ringId = ring.id; int stageId = GraphManager.getRingProducerId(gm, ringId); if (stageId>=0) { if (PronghornStage.supportsBatchedPublish(gm.stageIdToStage[stageId])) { Pipe.setMaxPublishBatchSize(ring); } } } } }
rootsTable, lastKnownRoot, count, rootMemberCounter, log, GraphManager.getRingConsumerId(graphManager, outputPipe.id),ensureAllPrevious); int consumerId = GraphManager.getRingConsumerId( graphManager, GraphManager.getOutputPipe( rootsTable, lastKnownRoot, count, rootMemberCounter, log, GraphManager.getRingConsumerId(graphManager, GraphManager.getOutputPipe(graphManager, stage, r).id), ensureAllPrevious);
public static synchronized int isStageInLoop(GraphManager graphManager, final int stageId) { if (null == graphManager.loopDetector) { graphManager.loopDetector = new StageDetector(); } graphManager.loopDetector.setTarget(graphManager, stageId); int c = GraphManager.getOutputPipeCount(graphManager, stageId); for(int i = 1; i<=c; i++) { int outPipeId = GraphManager.getOutputPipe(graphManager, stageId, c).id; visitDownstream(GraphManager.getRingConsumerId(graphManager, outPipeId), graphManager, cleanStageBitMap(graphManager), 0, graphManager.loopDetector); if ( graphManager.loopDetector.wasDetected()) { return graphManager.loopDetector.foundProducerStageId(); //will be -1 when no prodcuer was found. } } return -2; //This is not in a loop }
while ((pipeId = m.multOutputIds[outputPos++])>=0) { count++; int ringConsumerId = GraphManager.getRingConsumerId(m, pipeId); noConsumers = noConsumers & (ringConsumerId<0 || isStageTerminated(m,ringConsumerId));
int ringId = pipes[i].id; int consumerId = GraphManager.getRingConsumerId(graphManager, ringId); int producerId = GraphManager.getRingProducerId(graphManager, ringId);
int ringId = pipes[i].id; int consumerId = GraphManager.getRingConsumerId(graphManager, ringId); int producerId = GraphManager.getRingProducerId(graphManager, ringId);