/** * Gets a list of integer pair elements in range. * @param start value of the range (inclusive). * @param end value of the range (exclusive). * @return the list of elements. */ public static List getRangedNumList(final int start, final int end) { final List numList = new ArrayList<>(end - start); IntStream.range(start, end).forEach(number -> numList.add(Pair.of(number, number))); return numList; }
/** * Gets a list of integer pair elements in range. * @param start value of the range (inclusive). * @param end value of the range (exclusive). * @return the list of elements. */ public static List getRangedNumList(final int start, final int end) { final List numList = new ArrayList<>(end - start); IntStream.range(start, end).forEach(number -> numList.add(Pair.of(number, number))); return numList; }
public void onTaskScheduled(final String edgeId, final long srcIndex) { final Pair<String, Long> keyPair = Pair.of(edgeId, srcIndex); if (null != runtimeEdgeSrcIndexToLock.put(keyPair, new ReentrantLock())) { throw new IllegalStateException(keyPair.toString()); } if (null != runtimeEdgeSrcIndexToCondition.put(keyPair, runtimeEdgeSrcIndexToLock.get(keyPair).newCondition())) { throw new IllegalStateException(keyPair.toString()); } }
public void onTaskScheduled(final String edgeId, final long srcIndex) { final Pair<String, Long> keyPair = Pair.of(edgeId, srcIndex); if (null != runtimeEdgeSrcIndexToLock.put(keyPair, new ReentrantLock())) { throw new IllegalStateException(keyPair.toString()); } if (null != runtimeEdgeSrcIndexToCondition.put(keyPair, runtimeEdgeSrcIndexToLock.get(keyPair).newCondition())) { throw new IllegalStateException(keyPair.toString()); } }
synchronized void registerExecutor(final ExecutorRepresenter executor) { final String executorId = executor.getExecutorId(); if (executors.containsKey(executorId)) { throw new IllegalArgumentException("Duplicate executor: " + executor.toString()); } else { executors.put(executorId, Pair.of(executor, ExecutorState.RUNNING)); } }
synchronized void registerExecutor(final ExecutorRepresenter executor) { final String executorId = executor.getExecutorId(); if (executors.containsKey(executorId)) { throw new IllegalArgumentException("Duplicate executor: " + executor.toString()); } else { executors.put(executorId, Pair.of(executor, ExecutorState.RUNNING)); } }
@Override public boolean isReady(final PCollectionView view, final BoundedWindow window) { return window.maxTimestamp().getMillis() < curWatermark || inMemorySideInputs.containsKey(Pair.of(view, window)); }
@Nullable @Override public <T> T get(final PCollectionView<T> view, final BoundedWindow window) { // This gets called after isReady() final T sideInputData = (T) inMemorySideInputs.get(Pair.of(view, window)); return sideInputData == null // The upstream gave us an empty sideInput ? ((ViewFn<Object, T>) view.getViewFn()).apply(new CreateViewTransform.MultiView<T>(Collections.emptyList())) // The upstream gave us a concrete sideInput : sideInputData; }
/** * Action after task execution has been completed. * Note this method should not be invoked when the previous state of the task is ON_HOLD. * * @param executorId id of the executor. * @param taskId the ID of the task completed. */ private void onTaskExecutionComplete(final String executorId, final String taskId) { LOG.debug("{} completed in {}", taskId, executorId); executorRegistry.updateExecutor(executorId, (executor, state) -> { executor.onTaskExecutionComplete(taskId); return Pair.of(executor, state); }); }
/** * Action after task execution has been completed. * Note this method should not be invoked when the previous state of the task is ON_HOLD. * * @param executorId id of the executor. * @param taskId the ID of the task completed. */ private void onTaskExecutionComplete(final String executorId, final String taskId) { LOG.debug("{} completed in {}", taskId, executorId); executorRegistry.updateExecutor(executorId, (executor, state) -> { executor.onTaskExecutionComplete(taskId); return Pair.of(executor, state); }); }
synchronized void terminate() { for (final ExecutorRepresenter executor : getRunningExecutors()) { executor.shutDown(); executors.put(executor.getExecutorId(), Pair.of(executor, ExecutorState.TERMINATED)); } }
/** * Stores the side input in memory to be used with main inputs. * @param view of the side input. * @param sideInputElement to add. */ public void addSideInputElement(final PCollectionView<?> view, final WindowedValue<SideInputElement<?>> sideInputElement) { for (final BoundedWindow bw : sideInputElement.getWindows()) { inMemorySideInputs.put(Pair.of(view, bw), sideInputElement.getValue().getSideInputValue()); } }
synchronized void terminate() { for (final ExecutorRepresenter executor : getRunningExecutors()) { executor.shutDown(); executors.put(executor.getExecutorId(), Pair.of(executor, ExecutorState.TERMINATED)); } }
/** * Dynamic optimization method to process the dag with an appropriate pass, decided by the stats. * * @param originalPlan original physical execution plan. * @return the newly updated optimized physical plan. */ public static synchronized PhysicalPlan dynamicOptimization( final PhysicalPlan originalPlan, final Object dynOptData, final StageEdge targetEdge) { // Data for dynamic optimization used in DataSkewRuntimePass // is a map of <hash value, partition size>. final PhysicalPlan physicalPlan = new DataSkewRuntimePass() .apply(originalPlan, Pair.of(targetEdge, (Map<Object, Long>) dynOptData)); return physicalPlan; } }
private Map<String, List<OutputWriter>> getExternalAdditionalOutputMap( final IRVertex irVertex, final List<StageEdge> outEdgesToChildrenTasks, final IntermediateDataIOFactory intermediateDataIOFactory) { // Add all inter-task additional tags to additional output map. final Map<String, List<OutputWriter>> map = new HashMap<>(); outEdgesToChildrenTasks .stream() .filter(edge -> edge.getSrcIRVertex().getId().equals(irVertex.getId())) .filter(edge -> edge.getPropertyValue(AdditionalOutputTagProperty.class).isPresent()) .map(edge -> Pair.of(edge.getPropertyValue(AdditionalOutputTagProperty.class).get(), intermediateDataIOFactory.createWriter(taskId, edge))) .forEach(pair -> { map.putIfAbsent(pair.left(), new ArrayList<>()); map.get(pair.left()).add(pair.right()); }); return map; }
@Override public void onExecutorRemoved(final String executorId) { LOG.info("{} removed", executorId); blockManagerMaster.removeWorker(executorId); // These are tasks that were running at the time of executor removal. final Set<String> interruptedTasks = new HashSet<>(); executorRegistry.updateExecutor(executorId, (executor, state) -> { interruptedTasks.addAll(executor.onExecutorFailed()); return Pair.of(executor, ExecutorRegistry.ExecutorState.FAILED); }); // Blocks of the interrupted tasks are failed. interruptedTasks.forEach(blockManagerMaster::onProducerTaskFailed); // Retry the interrupted tasks (and required parents) retryTasksAndRequiredParents(interruptedTasks); // Trigger the scheduling of SHOULD_RETRY tasks in the earliest scheduleGroup doSchedule(); }
@Override public void onExecutorRemoved(final String executorId) { LOG.info("{} removed", executorId); blockManagerMaster.removeWorker(executorId); // These are tasks that were running at the time of executor removal. final Set<String> interruptedTasks = new HashSet<>(); executorRegistry.updateExecutor(executorId, (executor, state) -> { interruptedTasks.addAll(executor.onExecutorFailed()); return Pair.of(executor, ExecutorRegistry.ExecutorState.FAILED); }); // Blocks of the interrupted tasks are failed. interruptedTasks.forEach(blockManagerMaster::onProducerTaskFailed); // Retry the interrupted tasks (and required parents) retryTasksAndRequiredParents(interruptedTasks); // Trigger the scheduling of SHOULD_RETRY tasks in the earliest scheduleGroup doSchedule(); }
/** * (SYNCHRONIZATION) Called by task threads. * * @param runtimeEdge runtime edge * @param srcTaskIndex source task index * @return output contexts. */ public List<ByteOutputContext> getOutputContexts(final RuntimeEdge runtimeEdge, final long srcTaskIndex) { // First, initialize the pair key final Pair<String, Long> pairKey = Pair.of(runtimeEdge.getId(), srcTaskIndex); pipeContainer.putPipeListIfAbsent(pairKey, getNumOfPipeToWait(runtimeEdge)); // Then, do stuff return pipeContainer.getPipes(pairKey); // blocking call }
/** * (SYNCHRONIZATION) Called by task threads. * * @param runtimeEdge runtime edge * @param srcTaskIndex source task index * @return output contexts. */ public List<ByteOutputContext> getOutputContexts(final RuntimeEdge runtimeEdge, final long srcTaskIndex) { // First, initialize the pair key final Pair<String, Long> pairKey = Pair.of(runtimeEdge.getId(), srcTaskIndex); pipeContainer.putPipeListIfAbsent(pairKey, getNumOfPipeToWait(runtimeEdge)); // Then, do stuff return pipeContainer.getPipes(pairKey); // blocking call }
/** * (SYNCHRONIZATION) Called by network threads. * * @param outputContext output context * @throws InvalidProtocolBufferException protobuf exception */ public void onOutputContext(final ByteOutputContext outputContext) throws InvalidProtocolBufferException { final ControlMessage.PipeTransferContextDescriptor descriptor = ControlMessage.PipeTransferContextDescriptor.PARSER.parseFrom(outputContext.getContextDescriptor()); final long srcTaskIndex = descriptor.getSrcTaskIndex(); final String runtimeEdgeId = descriptor.getRuntimeEdgeId(); final int dstTaskIndex = (int) descriptor.getDstTaskIndex(); final int numPipeToWait = (int) descriptor.getNumPipeToWait(); final Pair<String, Long> pairKey = Pair.of(runtimeEdgeId, srcTaskIndex); // First, initialize the pair key pipeContainer.putPipeListIfAbsent(pairKey, numPipeToWait); // Then, do stuff pipeContainer.putPipe(pairKey, dstTaskIndex, outputContext); }