public ExecutionGraph getExecutionGraph() { return this.jobVertex.getGraph(); }
public ExecutionGraph getExecutionGraph() { return this.jobVertex.getGraph(); }
public ExecutionGraph getExecutionGraph() { return this.jobVertex.getGraph(); }
public ExecutionGraph getExecutionGraph() { return this.jobVertex.getGraph(); }
private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobVertices) { if (jobVertices.size() == 1) { ExecutionJobVertex jv = jobVertices.get(0); if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } return jv.getTaskVertices(); } else { ArrayList<ExecutionVertex> all = new ArrayList<>(); for (ExecutionJobVertex jv : jobVertices) { if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } all.addAll(Arrays.asList(jv.getTaskVertices())); } return all.toArray(new ExecutionVertex[all.size()]); } }
private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobVertices) { if (jobVertices.size() == 1) { ExecutionJobVertex jv = jobVertices.get(0); if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } return jv.getTaskVertices(); } else { ArrayList<ExecutionVertex> all = new ArrayList<>(); for (ExecutionJobVertex jv : jobVertices) { if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } all.addAll(Arrays.asList(jv.getTaskVertices())); } return all.toArray(new ExecutionVertex[all.size()]); } }
private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobVertices) { if (jobVertices.size() == 1) { ExecutionJobVertex jv = jobVertices.get(0); if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } return jv.getTaskVertices(); } else { ArrayList<ExecutionVertex> all = new ArrayList<>(); for (ExecutionJobVertex jv : jobVertices) { if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } all.addAll(Arrays.asList(jv.getTaskVertices())); } return all.toArray(new ExecutionVertex[all.size()]); } }
private ExecutionVertex[] collectExecutionVertices(List<ExecutionJobVertex> jobVertices) { if (jobVertices.size() == 1) { ExecutionJobVertex jv = jobVertices.get(0); if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } return jv.getTaskVertices(); } else { ArrayList<ExecutionVertex> all = new ArrayList<>(); for (ExecutionJobVertex jv : jobVertices) { if (jv.getGraph() != this) { throw new IllegalArgumentException("Can only use ExecutionJobVertices of this ExecutionGraph"); } all.addAll(Arrays.asList(jv.getTaskVertices())); } return all.toArray(new ExecutionVertex[all.size()]); } }
@Override public Void apply(StackTraceSample stackTraceSample, Throwable throwable) { synchronized (lock) { try { if (shutDown) { return null; } // Job finished, ignore. JobStatus jobState = vertex.getGraph().getState(); if (jobState.isGloballyTerminalState()) { LOG.debug("Ignoring sample, because job is in state " + jobState + "."); } else if (stackTraceSample != null) { OperatorBackPressureStats stats = createStatsFromSample(stackTraceSample); operatorStatsCache.put(vertex, stats); } else { LOG.debug("Failed to gather stack trace sample.", throwable); } } catch (Throwable t) { LOG.error("Error during stats completion.", t); } finally { pendingStats.remove(vertex); } return null; } }
@Override public Void apply(StackTraceSample stackTraceSample, Throwable throwable) { synchronized (lock) { try { if (shutDown) { return null; } // Job finished, ignore. JobStatus jobState = vertex.getGraph().getState(); if (jobState.isGloballyTerminalState()) { LOG.debug("Ignoring sample, because job is in state " + jobState + "."); } else if (stackTraceSample != null) { OperatorBackPressureStats stats = createStatsFromSample(stackTraceSample); operatorStatsCache.put(vertex, stats); } else { LOG.debug("Failed to gather stack trace sample.", throwable); } } catch (Throwable t) { LOG.error("Error during stats completion.", t); } finally { pendingStats.remove(vertex); } return null; } }
@Override public Void apply(StackTraceSample stackTraceSample, Throwable throwable) { synchronized (lock) { try { if (shutDown) { return null; } // Job finished, ignore. JobStatus jobState = vertex.getGraph().getState(); if (jobState.isGloballyTerminalState()) { LOG.debug("Ignoring sample, because job is in state " + jobState + "."); } else if (stackTraceSample != null) { OperatorBackPressureStats stats = createStatsFromSample(stackTraceSample); operatorStatsCache.put(vertex, stats); } else { LOG.debug("Failed to gather stack trace sample.", throwable); } } catch (Throwable t) { LOG.error("Error during stats completion.", t); } finally { pendingStats.remove(vertex); } return null; } }
@Override public Void apply(StackTraceSample stackTraceSample, Throwable throwable) { synchronized (lock) { try { if (shutDown) { return null; } // Job finished, ignore. JobStatus jobState = vertex.getGraph().getState(); if (jobState.isGloballyTerminalState()) { LOG.debug("Ignoring sample, because job is in state " + jobState + "."); } else if (stackTraceSample != null) { OperatorBackPressureStats stats = createStatsFromSample(stackTraceSample); operatorStatsCache.put(vertex, stats); } else { LOG.debug("Failed to gather stack trace sample.", throwable); } } catch (Throwable t) { LOG.error("Error during stats completion.", t); } finally { pendingStats.remove(vertex); } return null; } }
getGraph().getGraphManager().notifyInputSplitsCreated(getJobVertexId(), inputSplitsMap); } finally { currentThread.setContextClassLoader(oldContextClassLoader);
private int calculateTaskExtraManagedMemory() { Configuration config = getJobVertex().getGraph().getJobManagerConfiguration(); // Calculates managed memory for external result partition. BlockingShuffleType shuffleType = BlockingShuffleType.getBlockingShuffleTypeFromConfiguration(config, LOG); int numExternalResultPartitions = 0; for (IntermediateResultPartition irp : getProducedPartitions().values()) { if (shuffleType == BlockingShuffleType.YARN && irp.getIntermediateResult().getResultType().isBlocking()) { numExternalResultPartitions++; } } int mapOutputMemoryInMB = config.getInteger(TaskManagerOptions.TASK_MANAGER_OUTPUT_MEMORY_MB); return mapOutputMemoryInMB * numExternalResultPartitions; }
@VisibleForTesting int calculateTaskNetworkMemory() { Configuration config = jobVertex.getGraph().getJobManagerConfiguration();