@SuppressWarnings("unchecked") public static <S extends MessageSchema<S>> Pipe<S> getOutputPipe(GraphManager m, int stageId, int ordinalOutput) { assert(ordinalOutput>0); assert(ordinalOutput<=getOutputPipeCount(m,stageId)); return m.pipeIdToPipe[m.multOutputIds[m.stageIdToOutputsBeginIdx[stageId]+ordinalOutput-1]]; }
private static boolean isInputLocal(int startIdx, int stopIdx, GraphManager gm, PronghornStage[] stages, int[] script, int goalId) { //scan for an output which matches this goal Id for(int i = startIdx; i<=stopIdx; i++) { int stageId = stages[script[i]].stageId; int outC = GraphManager.getOutputPipeCount(gm, stageId); for(int k = 1; k <= outC; k++) { if (goalId == GraphManager.getOutputPipe(gm, stageId, k).id) { return true; } } } return false; }
public static void addPublishFromListener(GraphManager graphManager, PronghornStage pronghornStage, PipePublishListener listener) { int c = getOutputPipeCount(graphManager, pronghornStage.stageId); for(int i=1; i<=c; i++) { Pipe.addPubListener(getOutputPipe(graphManager, pronghornStage.stageId, i), listener); } }
public static void appendOutputs(GraphManager graphManager, Appendable target, PronghornStage pronghornStage) { int c = GraphManager.getOutputPipeCount(graphManager, pronghornStage.stageId); for(int i = 1; i<=c; i++) { int id = GraphManager.getOutputPipe(graphManager, pronghornStage.stageId, i).id; Appendables.appendValue(target, i==1?"":" ,", id); } }
public static void removePublishFromListener(GraphManager graphManager, PronghornStage pronghornStage, PipePublishListener listener) { int c = getOutputPipeCount(graphManager, pronghornStage.stageId); for(int i=1; i<=c; i++) { Pipe.removePubListener(getOutputPipe(graphManager, pronghornStage.stageId, i), listener); } }
public static RunningStdDev stdDevPipesPerStage(GraphManager m) { if (null == m.stdDevPipes) { m.stdDevPipes = new RunningStdDev(); int i = m.stageIdToStage.length; while (--i>=0) { PronghornStage stage = m.stageIdToStage[i]; if (null!=stage) { int sample = getInputPipeCount(m, stage.stageId)+ getOutputPipeCount(m, stage.stageId); RunningStdDev.sample(m.stdDevPipes, sample); } } } return m.stdDevPipes; }
private boolean countParallelProducers(int consumerId, int producerId, GraphManager graphManager, Pipe p) { int noMatchCount = 0; int proOutCount = GraphManager.getOutputPipeCount(graphManager, producerId); while (--proOutCount>=0) { //all the other output coming from the producer Pipe outputPipe = GraphManager.getOutputPipe(graphManager, producerId, 1+proOutCount); //is this the same schema as the pipe in question. if (Pipe.isForSameSchema(outputPipe, p)) { //determine if they are consumed by the same place or not int conId = GraphManager.getRingConsumerId(graphManager, outputPipe.id); if (consumerId != conId) { //only count if they are not consumed at the same place noMatchCount++; } //else if they are consumed by the same place then the pipes are just for parallel storage not parallel compute } } return noMatchCount>=1; }
private static Pipe[] buildProducersPipes(int count, int indexesIdx, int outputIdx, final int[] indexes, final PronghornStage[] stages, final GraphManager graphManager) { while (indexesIdx < indexes.length) { int outputCount = GraphManager.getOutputPipeCount(graphManager, stages[indexes[indexesIdx]].stageId); while (outputIdx <= outputCount) { Pipe pipe = GraphManager.getOutputPipe(graphManager, stages[indexes[indexesIdx]], outputIdx); //is the consumer of this pipe inside the graph? int consumerId = GraphManager.getRingConsumerId(graphManager, pipe.id); int k = stages.length; while (--k >= 0) { if (stages[k].stageId == consumerId) { Pipe[] result = buildProducersPipes(count + 1, indexesIdx, outputIdx + 1, indexes, stages, graphManager); result[count] = pipe; return result; } } outputIdx++; } outputIdx = 1; indexesIdx++; } return new Pipe[count]; }
private static Pipe[] buildProducersPipes(int count, int indexesIdx, int outputIdx, final int[] indexes, final PronghornStage[] stages, final GraphManager graphManager) { while(indexesIdx<indexes.length) { int outputCount = GraphManager.getOutputPipeCount(graphManager, stages[indexes[indexesIdx]].stageId); while(outputIdx<=outputCount) { Pipe pipe = GraphManager.getOutputPipe(graphManager, stages[indexes[indexesIdx]], outputIdx); //is the consumer of this pipe inside the graph? int consumerId = GraphManager.getRingConsumerId(graphManager, pipe.id); int k = stages.length; while (--k>=0) { if (stages[k].stageId==consumerId) { Pipe[] result = buildProducersPipes(count+1, indexesIdx, outputIdx+1, indexes, stages, graphManager); result[count] = pipe; return result; } } outputIdx++; } outputIdx=1; indexesIdx++; } return new Pipe[count]; }
private void processException(PronghornStage stage, Exception e) { recordTheException(stage, e, this); ////////////////////// //check the output pipes to ensure no writes are in progress ///////////////////// int c = GraphManager.getOutputPipeCount(graphManager, stage.stageId); for(int i=1; i<=c; i++) { if (Pipe.isInBlobFieldWrite((Pipe<?>) GraphManager.getOutputPipe(graphManager, stage.stageId, i))) { //we can't recover from this exception because write was left open.... shutdown(); break; } } }
private static void visitDownstream(int stageId, GraphManager graphManager, long[] bitMap, int depth, GraphVisitor visitor) { assert(stageId<=graphManager.stageCounter.get()) : "This stage id is out of range"; if (stageId<0) { //nothing to visit return; } int mapIndex = stageId>>6;//divide by 64 int mapBit = 1<<(stageId&0x3F); if ((bitMap[mapIndex]&mapBit)!=0) { return;//we have encountered some place we have already been. } else { //mark this as a new place we have been bitMap[mapIndex] |= mapBit; //now call everyone downstream //visit children only if visitor returns true if (visitor.visit(graphManager, stageId, depth)) { int c = GraphManager.getOutputPipeCount(graphManager, stageId); for(int i = 1; i<=c; i++) { int outPipeId = GraphManager.getOutputPipe(graphManager, stageId, c).id; visitDownstream(GraphManager.getRingConsumerId(graphManager, outPipeId), graphManager, bitMap, depth+1, visitor); } } } }
Pipe p, boolean useMoreThreads) { int countOfHeavyComputeConsumers = 0; final int totalOutputsCount = GraphManager.getOutputPipeCount(graphManager, producerId); if (totalOutputsCount<=1) { return false;
private void add(PronghornStage[] pronghornStages, PronghornStage stage, final int root, final GraphManager graphManager, final IntHashTable rootsTable) { int i = 0; while (i<pronghornStages.length && pronghornStages[i]!=null) { if (pronghornStages[i]==stage) { return;//already added } i++; } //now add the new stage at index i pronghornStages[i]=stage; //Recursively add the ones under the same root. int outputCount = GraphManager.getOutputPipeCount(graphManager, stage.stageId); for(int r = 1; r<=outputCount; r++) { Pipe outputPipe = GraphManager.getOutputPipe(graphManager, stage, r); int consumerId = GraphManager.getRingConsumerId(graphManager, outputPipe.id); //this exists and has the same root so add it if (consumerId>=0 && rootId(consumerId, rootsTable)==root) { add(pronghornStages, GraphManager.getStage(graphManager, consumerId), root, graphManager, rootsTable); } } }
int thresholdToNeverCombine = (int) (RunningStdDev.mean(pipesPerStage)+ (2*RunningStdDev.stdDeviation(pipesPerStage))); if ((GraphManager.getInputPipeCount(graphManager,consumerId) +GraphManager.getOutputPipeCount(graphManager, consumerId)) > thresholdToNeverCombine) { return false; +GraphManager.getOutputPipeCount(graphManager, producerId)) > thresholdToNeverCombine) { return false;
private static void fuzzTestStage(long testDuration, Random random, GraphManager gm, PronghornStage stageToTest) { int c = GraphManager.getOutputPipeCount(gm, stageToTest.stageId); Pipe[] outputPipes = new Pipe[c]; for(int i=1; i<=c; i++) {
public static synchronized int isStageInLoop(GraphManager graphManager, final int stageId) { if (null == graphManager.loopDetector) { graphManager.loopDetector = new StageDetector(); } graphManager.loopDetector.setTarget(graphManager, stageId); int c = GraphManager.getOutputPipeCount(graphManager, stageId); for(int i = 1; i<=c; i++) { int outPipeId = GraphManager.getOutputPipe(graphManager, stageId, c).id; visitDownstream(GraphManager.getRingConsumerId(graphManager, outPipeId), graphManager, cleanStageBitMap(graphManager), 0, graphManager.loopDetector); if ( graphManager.loopDetector.wasDetected()) { return graphManager.loopDetector.foundProducerStageId(); //will be -1 when no prodcuer was found. } } return -2; //This is not in a loop }
&& (GraphManager.getOutputPipeCount(graphManager, producerId)>4) ) { return false; && (GraphManager.getOutputPipeCount(graphManager, producerId)>2) ) {
int outputCount = GraphManager.getOutputPipeCount(graphManager, stage.stageId);
final int outputPipeCount = GraphManager.getOutputPipeCount(m, producer); int sum=0; int lastPipeId=-1; final int width = GraphManager.getOutputPipeCount(m, producer); int maxPctFull = 0; long sumTraffic = 0;