@Override public int getProducerId(int pipeId) { return GraphManager.getRingProducerId(graphManager, pipeId); }
public static int getRingProducerStageId(GraphManager gm, int ringId) { return getRingProducerId(gm, ringId); }
public static PronghornStage getRingProducer(GraphManager gm, int ringId) { int stageId = getRingProducerId(gm, ringId); if (stageId<0) { throw new UnsupportedOperationException("Can not find input stage writing to pipe "+ringId+". Check graph construction."); } return gm.stageIdToStage[stageId]; }
public static boolean isProducerTerminated(GraphManager m, int ringId) { int producerStageId = getRingProducerId(m, ringId); return producerStageId<0 || stageStateArray(m)[producerStageId] == GraphManagerStageStateData.STAGE_TERMINATED; }
private static float computeWeightBetweenStages(GraphManager m, int consumerStage, int producerStage) { float weight = 0f; Pipe[] pipes = m.pipeIdToPipe; int p = pipes.length; while (--p >= 0) { if (null != pipes[p]) { if (producerStage == GraphManager.getRingProducerId(m, pipes[p].id) && consumerStage == GraphManager.getRingConsumerId(m, pipes[p].id)) { weight += 2; } else if (consumerStage == GraphManager.getRingProducerId(m, pipes[p].id) && producerStage == GraphManager.getRingConsumerId(m, pipes[p].id)) { weight += 2; } } } return weight; }
private static boolean isSameSource(GraphManager m, int consumer, int a, int b) { Pipe<?> inputPipe1 = GraphManager.getInputPipe(m, consumer, a); Pipe<?> inputPipe2 = GraphManager.getInputPipe(m, consumer, b); int prod1 = GraphManager.getRingProducerId(m, inputPipe1.id); int prod2 = GraphManager.getRingProducerId(m, inputPipe2.id); //System.out.println(prod1+" "+prod2+" "+inputPipe1+" "+inputPipe2); return (prod1==prod2 && Pipe.isForSameSchema(inputPipe1, inputPipe2)); }
private static <T extends MessageSchema<T>> Pipe<T>[] pipesOfTypeWithNoProducer(int count, int p, GraphManager gm, Class<T> targetSchemaClazz) { //pass one to count all the instances while (--p>=0) { Pipe tp = gm.pipeIdToPipe[p]; if ((null != tp) && Pipe.isForSchema(tp, targetSchemaClazz) && (getRingProducerId(gm, tp.id)==-1)) { Pipe<T>[] result = pipesOfTypeWithNoProducer(count+1,p,gm,targetSchemaClazz); result[(result.length-1)-count] = tp; return result; } } return (0==count) ? EMPTY_PIPE_ARRAY : new Pipe[count]; }
private boolean countParallelConsumers(int consumerId, int producerId, GraphManager graphManager, Pipe p) { int noMatchCount = 0; int conInCount = GraphManager.getInputPipeCount(graphManager, consumerId); while (--conInCount>=0) { //all the other inputs going into the consumer Pipe inputPipe = GraphManager.getInputPipe(graphManager, consumerId, 1+conInCount); //is this the same schema as the pipe in question. if (Pipe.isForSameSchema(inputPipe, p)) { //determine if they are coming from the same place or not int prodId = GraphManager.getRingProducerId(graphManager, inputPipe.id); if (producerId != prodId) { //only count if they are NOT coming from the same place noMatchCount++; } //else if they are all from the samme place then pipes are used here for holding parallel work not for CPU intensive activities } } return noMatchCount>=1; }
private static Pipe[] buildInternalPipes(int count, int stageIdx, int inputIdx, final PronghornStage[] stages, final GraphManager graphManager) { while(stageIdx<stages.length) { int inputCount = GraphManager.getInputPipeCount(graphManager, stages[stageIdx]); while(inputIdx<=inputCount) { Pipe pipe = GraphManager.getInputPipe(graphManager, stages[stageIdx], inputIdx); int producerId = GraphManager.getRingProducerId(graphManager, pipe.id); int k = stages.length; while (--k>=0) { if (stages[k].stageId==producerId) { Pipe[] result = buildInternalPipes(count+1, stageIdx, inputIdx+1, stages, graphManager); result[count] = pipe; return result; } } inputIdx++; } inputIdx=1; stageIdx++; } return new Pipe[count]; }
int producerStageId = GraphManager.getRingProducerId(graphManager, inputPipe.id); int producerStageId = GraphManager.getRingProducerId(graphManager, inputPipe.id);
private static void regOutput(GraphManager gm, Pipe output, int stageId) { if (null!=output) { int outputId = output.id; Pipe.structRegistry(output,gm.recordTypeData); try { gm.ringIdToStages = setValue(gm.ringIdToStages, (outputId*2) , stageId, gm.stageIdToStage[stageId]); //source +0 then target +1 gm.pipeIdToPipe = setValue(gm.pipeIdToPipe, outputId, output); gm.multOutputIds = setValue(gm.multOutputIds, gm.topOutput++, outputId, output); } catch (UnsupportedOperationException uoe) { output.creationStack(); logger.info("\nRegister: "+output); throw uoe; } assert(stageId == getRingProducerId(gm, outputId)); } }
int producerId = GraphManager.getRingProducerId(graphManager, pipe.id);
private static boolean hasNoInternalInputs( GraphManager graphManager, PronghornStage stage, IntHashTable rootsTable, IntArrayHolder lastKnownRoot) { boolean result = true; final int root = rootId(stage.stageId, rootsTable, lastKnownRoot); int inC = GraphManager.getInputPipeCount(graphManager, stage.stageId); for(int i=1; i<=inC; i++) { //if we find an input in the same root then we do have inputs int prodId = GraphManager.getRingProducerId(graphManager, GraphManager.getInputPipe(graphManager, stage.stageId, i).id); if (root == rootId(prodId, rootsTable, lastKnownRoot)) { result = false; } } return result; }
int prodId = GraphManager.getRingProducerId(graphManager, inputPipe.id); if ( (totalInputsCount-1) == conInCount &&
for(int j = 1; j<= inC; j++) { int prodId = GraphManager.getRingProducerId(graphManager, GraphManager.getInputPipeId(graphManager, consumerId, j));
public int weight(Pipe p) { if (weights[p.id]==0) { int result = (int)p.config().slabBits(); //returns the max pipe length from this pipe or any of the pipes that feed its producer. //if this value turns out to be large then we should probably not join these two stages. int producerId = GraphManager.getRingProducerId(graphManager, p.id); if (producerId>=0) { PronghornStage producer = GraphManager.getStage(graphManager, producerId); int count = GraphManager.getInputPipeCount(graphManager, producer); while (--count>=0) { Pipe inputPipe = GraphManager.getInputPipe(graphManager, producer, count+1); result = Math.max(result, inputPipe.config().slabBits()); } } else { //no producer found, an external thread must be pushing data into this, there is nothing to combine it with } weights[p.id] = result; } return weights[p.id]; }
public int weight(Pipe p) { if (weights[p.id]==0) { int result = (int)p.config().slabBits(); //returns the max pipe length from this pipe or any of the pipes that feed its producer. //if this value turns out to be large then we should probably not join these two stages. int producerId = GraphManager.getRingProducerId(graphManager, p.id); if (producerId>=0) { PronghornStage producer = GraphManager.getStage(graphManager, producerId); int count = GraphManager.getInputPipeCount(graphManager, producer); while (--count>=0) { Pipe inputPipe = GraphManager.getInputPipe(graphManager, producer, count); result = Math.max(result, inputPipe.config().slabBits()); } } else { //no producer found, an external thread must be pushing data into this, there is nothing to combine it with } weights[p.id] = result; } return weights[p.id]; }
public static void enableBatching(GraphManager gm) { int j = gm.pipeIdToPipe.length; while (--j>=0) { Pipe ring = gm.pipeIdToPipe[j]; //never enable batching on the monitor rings if (null!=ring && !ringHoldsMonitorData(gm, ring) ) { int ringId1 = ring.id; int stageId1 = GraphManager.getRingConsumerId(gm, ringId1); if (stageId1>=0) { if (PronghornStage.supportsBatchedRelease(gm.stageIdToStage[stageId1])) { Pipe.setMaxReleaseBatchSize(ring); } } int ringId = ring.id; int stageId = GraphManager.getRingProducerId(gm, ringId); if (stageId>=0) { if (PronghornStage.supportsBatchedPublish(gm.stageIdToStage[stageId])) { Pipe.setMaxPublishBatchSize(ring); } } } } }
int producerId = GraphManager.getRingProducerId(graphManager, ringId);
int producerId = GraphManager.getRingProducerId(graphManager, ringId);