@SuppressWarnings("unchecked") public static <S extends MessageSchema<S>> Pipe<S> getInputPipe(GraphManager m, PronghornStage stage, int ordinalInput) { return getInputPipe(m, stage.stageId, ordinalInput); }
public boolean isContentForStage(PronghornStage stage) { int inC = GraphManager.getInputPipeCount(graphManager, stage.stageId); for(int k = 1; k <= inC; k++) { if (Pipe.contentRemaining((Pipe<?>) GraphManager.getInputPipe(graphManager, stage.stageId, k) ) != 0) { return true; } } return false; }
public static void appendInputs(GraphManager graphManager, Appendable target, PronghornStage pronghornStage) { int c = GraphManager.getInputPipeCount(graphManager, pronghornStage.stageId); for(int i = 1; i<=c; i++) { int id = GraphManager.getInputPipe(graphManager, pronghornStage.stageId, i).id; Appendables.appendValue(target, i==1?"":" ,", id); } }
public static <S extends MessageSchema<S>> Pipe<S> getInputPipe(GraphManager m, int stageId) { return getInputPipe(m, stageId, 1); }
private void recordTheException(final PronghornStage stage, Throwable t) { synchronized(this) { if (null==firstException) { firstException = t; } } logger.error("Stacktrace",t); if (null==stage) { logger.error("Stage was never initialized"); } else { int inputcount = GraphManager.getInputPipeCount(graphManager, stage); logger.error("Unexpected error in stage "+stage.stageId+" "+stage.getClass().getSimpleName()+" inputs:"+inputcount); int i = inputcount; while (--i>=0) { logger.error("left input pipe in state:"+ GraphManager.getInputPipe(graphManager, stage, i+1)); } GraphManager.shutdownNeighborRings(graphManager, stage); } } };
public static <S extends MessageSchema<S>> Pipe<S> getInputPipe(GraphManager m, PronghornStage stage) { return getInputPipe(m, stage, 1); }
private boolean countParallelConsumers(int consumerId, int producerId, GraphManager graphManager, Pipe p) { int noMatchCount = 0; int conInCount = GraphManager.getInputPipeCount(graphManager, consumerId); while (--conInCount>=0) { //all the other inputs going into the consumer Pipe inputPipe = GraphManager.getInputPipe(graphManager, consumerId, 1+conInCount); //is this the same schema as the pipe in question. if (Pipe.isForSameSchema(inputPipe, p)) { //determine if they are coming from the same place or not int prodId = GraphManager.getRingProducerId(graphManager, inputPipe.id); if (producerId != prodId) { //only count if they are NOT coming from the same place noMatchCount++; } //else if they are all from the samme place then pipes are used here for holding parallel work not for CPU intensive activities } } return noMatchCount>=1; }
private static boolean isSameSource(GraphManager m, int consumer, int a, int b) { Pipe<?> inputPipe1 = GraphManager.getInputPipe(m, consumer, a); Pipe<?> inputPipe2 = GraphManager.getInputPipe(m, consumer, b); int prod1 = GraphManager.getRingProducerId(m, inputPipe1.id); int prod2 = GraphManager.getRingProducerId(m, inputPipe2.id); //System.out.println(prod1+" "+prod2+" "+inputPipe1+" "+inputPipe2); return (prod1==prod2 && Pipe.isForSameSchema(inputPipe1, inputPipe2)); }
private static Pipe[] buildInternalPipes(int count, int stageIdx, int inputIdx, final PronghornStage[] stages, final GraphManager graphManager) { while(stageIdx<stages.length) { int inputCount = GraphManager.getInputPipeCount(graphManager, stages[stageIdx]); while(inputIdx<=inputCount) { Pipe pipe = GraphManager.getInputPipe(graphManager, stages[stageIdx], inputIdx); int producerId = GraphManager.getRingProducerId(graphManager, pipe.id); int k = stages.length; while (--k>=0) { if (stages[k].stageId==producerId) { Pipe[] result = buildInternalPipes(count+1, stageIdx, inputIdx+1, stages, graphManager); result[count] = pipe; return result; } } inputIdx++; } inputIdx=1; stageIdx++; } return new Pipe[count]; }
while(inputIdx<=inputCount) { Pipe pipe = GraphManager.getInputPipe(graphManager, stages[stageIdx], inputIdx);
while (--conInCount>=0) { Pipe inputPipe = GraphManager.getInputPipe(graphManager, consumerId, 1+conInCount);
private static boolean hasNoInternalInputs( GraphManager graphManager, PronghornStage stage, IntHashTable rootsTable, IntArrayHolder lastKnownRoot) { boolean result = true; final int root = rootId(stage.stageId, rootsTable, lastKnownRoot); int inC = GraphManager.getInputPipeCount(graphManager, stage.stageId); for(int i=1; i<=inC; i++) { //if we find an input in the same root then we do have inputs int prodId = GraphManager.getRingProducerId(graphManager, GraphManager.getInputPipe(graphManager, stage.stageId, i).id); if (root == rootId(prodId, rootsTable, lastKnownRoot)) { result = false; } } return result; }
Pipe<?> basePipe = GraphManager.getInputPipe(gm, stage.stageId, 1); allPipesAreForSameSchema |= Pipe.isForSameSchema(basePipe, (Pipe<?>) GraphManager.getInputPipe(gm, stage.stageId, x));
private static void recordRunResults( ScriptedNonThreadScheduler that, final boolean recordTime, int inProgressIdx, long start, long SLAStart, PronghornStage stage) { if (recordTime) { long now = System.nanoTime(); //this takes time, avoid if possible if (!GraphManager.accumRunTimeNS(that.graphManager, stage.stageId, now-start, now)){ assert(reportLowAccuracyClock(that)); } //No need to run on every call, we run 1 out of every 4 if (0== (0x3 & that.msgConsumerTrigger++)) { int c = GraphManager.getInputPipeCount(that.graphManager, stage.stageId); for(int i = 1; i<=c ;i++) { Pipe<?> pipe = GraphManager.getInputPipe(that.graphManager, stage.stageId, i); pipe.markConsumerPassDone(); } } } }
} else { for(int k = 1; k <= inC1; k++) { int id = GraphManager.getInputPipe(gm, stageId, k).id;
public int weight(Pipe p) { if (weights[p.id]==0) { int result = (int)p.config().slabBits(); //returns the max pipe length from this pipe or any of the pipes that feed its producer. //if this value turns out to be large then we should probably not join these two stages. int producerId = GraphManager.getRingProducerId(graphManager, p.id); if (producerId>=0) { PronghornStage producer = GraphManager.getStage(graphManager, producerId); int count = GraphManager.getInputPipeCount(graphManager, producer); while (--count>=0) { Pipe inputPipe = GraphManager.getInputPipe(graphManager, producer, count); result = Math.max(result, inputPipe.config().slabBits()); } } else { //no producer found, an external thread must be pushing data into this, there is nothing to combine it with } weights[p.id] = result; } return weights[p.id]; }
public int weight(Pipe p) { if (weights[p.id]==0) { int result = (int)p.config().slabBits(); //returns the max pipe length from this pipe or any of the pipes that feed its producer. //if this value turns out to be large then we should probably not join these two stages. int producerId = GraphManager.getRingProducerId(graphManager, p.id); if (producerId>=0) { PronghornStage producer = GraphManager.getStage(graphManager, producerId); int count = GraphManager.getInputPipeCount(graphManager, producer); while (--count>=0) { Pipe inputPipe = GraphManager.getInputPipe(graphManager, producer, count+1); result = Math.max(result, inputPipe.config().slabBits()); } } else { //no producer found, an external thread must be pushing data into this, there is nothing to combine it with } weights[p.id] = result; } return weights[p.id]; }
Pipe inPipe = GraphManager.getInputPipe(graphManager, consumerId, i); int inProd = GraphManager.getRingProducerStageId(graphManager, inPipe.id); int tempRootId = rootId(inProd, rootsTable, lastKnownRoot);
Pipe[] inputPipes = new Pipe[c]; for(int i=1; i<=c; i++) { inputPipes[i-1] = GraphManager.getInputPipe(gm, stageToTest.stageId, i);
int ringProducerStageId = GraphManager.getRingProducerStageId(graphManager, GraphManager.getInputPipe(graphManager, stage, j).id); if (rootId(ringProducerStageId, rootsTable) != root || GraphManager.isStageInLoop(graphManager, stage.stageId)>=-1