public StageScheduler(GraphManager graphManager) { GraphManager.disableMutation(graphManager); this.graphManager = graphManager; assert(initThreadChecking(graphManager)); }
public NonThreadScheduler(GraphManager graphManager, PronghornStage[] stages, String name) { super(graphManager); this.stages = stages; this.graphManager = graphManager; this.name = name; this.inLargerScheduler = false; recordTime = GraphManager.isTelemetryEnabled(graphManager); if (recordTime) { didWorkMonitor = new DidWorkMonitor(); } }
public NonThreadScheduler(GraphManager graphManager) { super(graphManager); this.stages = GraphManager.allStages(graphManager); this.graphManager = graphManager; this.inLargerScheduler = false; recordTime = GraphManager.isTelemetryEnabled(graphManager); if (recordTime) { didWorkMonitor = new DidWorkMonitor(); } }
/** * Use the default scheduler with the passed in GraphManager to start */ void startup() { StageScheduler.defaultScheduler(gm).startup(); }
public static void register(GraphManager gm, PronghornStage stage, Pipe input, Pipe output) { synchronized(gm.lock) { int stageId = beginStageRegister(gm, stage); setStateToNew(gm, stageId); //loop over inputs regInput(gm, input, stageId); //loop over outputs regOutput(gm, output, stageId); endStageRegister(gm, stage); } }
private static int[] buildProducersList(int count, int idx, final GraphManager graphManager, PronghornStage[] stages) { //skip over the non producers while (idx<stages.length) { if (null!=GraphManager.getNota(graphManager, stages[idx].stageId, GraphManager.PRODUCER, null) || (0==GraphManager.getInputPipeCount(graphManager, stages[idx])) ) { int[] result = buildProducersList(count+1, idx+1, graphManager, stages); result[count] = idx; return result; } idx++; } return new int[count]; }
public void requestShutdown() { if (!GraphManager.isStageShuttingDown(graphManager, stageId)) { GraphManager.setStateToStopping(graphManager, stageId); } }
private static void setStateToNew(GraphManager gm, int stageId) { synchronized(gm.stageStateData.lock) { gm.stageStateData.stageStateArray = setValue(stageStateArray(gm), stageId, GraphManagerStageStateData.STAGE_NEW); } }
public ScriptedFixedThreadsScheduler(GraphManager graphManager) { //this is often very optimal since we have enough granularity to swap work but we do not //have so many threads that it overwhelms the operating system context switching this(graphManager, CoresUtil.availableProcessors()*2); }
@Override public void shutdown() { int i = ntsArray.length; while (--i>=0) { if (null != ntsArray[i]) { if (!ScriptedNonThreadScheduler.isShutdownRequested(ntsArray[i])) { ntsArray[i].shutdown(); } } } }
@Override public boolean checkForException() { int i = ntsArray.length; while (--i>=0) { ntsArray[i].checkForException(); } return true; }
public static Comparator<? super Pipe> joinFirstComparator(GraphManager m) { assert(!m.enableMutation) : "must not call until we are done with contruction"; if (null==m.joinFirstComparator) { m.joinFirstComparator = new JoinFirstComparator(m); } return m.joinFirstComparator; }
private static void accumPositiveTime(GraphManager graphManager, int stageId, long duration, long now) { long[] stageLastTimeNsLocal = graphManager.stageLastTimeNs; long last = stageLastTimeNsLocal[stageId]; if (last>0 && last<now) { accumDuration(graphManager, stageId, duration, now-last); //this total duration is also used as the baseline for the histogram of elapsed time graphManager.stageRunNS[stageId] += duration; } //we have the running pct since startup but we need a recent rolling value. stageLastTimeNsLocal[stageId] = now; }
private static boolean mustShowOnTelemetry(GraphManager m, PronghornStage stage) { return null!=stage && !(stage instanceof PipeMonitorCollectorStage) && !(stage instanceof PipeMonitorStage) && !stageForMonitorData(m,stage); }
public NonThreadScheduler(GraphManager graphManager, PronghornStage[] stages, String name, boolean isInLargerScheduler) { super(graphManager); this.stages = stages; this.graphManager = graphManager; this.name = name; this.inLargerScheduler = isInLargerScheduler; recordTime = GraphManager.isTelemetryEnabled(graphManager); if (recordTime) { didWorkMonitor = new DidWorkMonitor(); } }
public static void setStateToStopping(GraphManager gm, int stageId) { synchronized(gm.stageStateData.lock) { gm.stageStateData.stageStateArray = setValue(stageStateArray(gm), stageId, GraphManagerStageStateData.STAGE_STOPPING); } }
public ClientHTTPRequestDataGeneratorStage(GraphManager gm, Pipe<NetPayloadSchema> output, int iterations, CharSequence[] paths) { super(gm, NONE, output); this.output = output; this.iterations = iterations; this.count = iterations; this.pos = 0; this.paths = paths; GraphManager.addNota(gm, GraphManager.PRODUCER, GraphManager.PRODUCER, this); verb = "GET "; // .5M msg/sec //verb = "HEAD "; //1.3M msg/sec }