@Override public void visitPrimitiveTransform(Node node) { if (freedNodes.contains(node.getEnclosingNode())) { freedNodes.add(node); } else if (override.getMatcher().matches(node.toAppliedPTransform(getPipeline()))) { matches.add(node); freedNodes.add(node); } } });
@Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { AppliedPTransform<?, ?, ?> appliedPTransform = node.toAppliedPTransform(getPipeline()); if (hasUnboundedOutput(appliedPTransform)) { Class<? extends PTransform> transformClass = node.getTransform().getClass(); LOG.info("Found {}. Switching to streaming execution.", transformClass); translationMode = TranslationMode.STREAMING; } }
@Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { LOG.debug("visiting transform {}", node.getTransform()); PTransform transform = node.getTransform(); TransformTranslator translator = getTransformTranslator(transform.getClass()); if (null == translator) { throw new UnsupportedOperationException("no translator registered for " + transform); } translationContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); translator.translate(transform, translationContext); }
@Override public void leaveCompositeTransform(Node node) { if (node.getTransform() instanceof Combine.PerKey) { checkState(combine.get() == null); combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline())); } } });
@Override public void leaveCompositeTransform(Node node) { if (node.getTransform() instanceof Combine.PerKey) { checkState(combine.get() == null); combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline())); } } });
@Override public void leaveCompositeTransform(Node node) { if (node.getTransform() instanceof Combine.PerKey) { checkState(combine.get() == null); combine.set((AppliedPTransform) node.toAppliedPTransform(getPipeline())); } } });
private AppliedPTransform<?, ?, ?> getAppliedTransform(TransformHierarchy.Node node) { @SuppressWarnings({"rawtypes", "unchecked"}) AppliedPTransform<?, ?, ?> application = node.toAppliedPTransform(getPipeline()); return application; }
private <T extends PTransform<?, ?>> boolean applyCanTranslate( PTransform<?, ?> transform, TransformHierarchy.Node node, StreamTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator; streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); return typedTranslator.canTranslate(typedTransform, streamingContext); }
private <T extends PTransform<?, ?>> void applyStreamingTransform( PTransform<?, ?> transform, TransformHierarchy.Node node, StreamTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator; // create the applied PTransform on the streamingContext streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); typedTranslator.translateNode(typedTransform, streamingContext); }
private <T extends PTransform<?, ?>> void applyBatchTransform( PTransform<?, ?> transform, TransformHierarchy.Node node, BatchTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator; // create the applied PTransform on the batchContext batchContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); typedTranslator.translateNode(typedTransform, batchContext); }
private <T extends PTransform<?, ?>> boolean applyCanTranslate( PTransform<?, ?> transform, TransformHierarchy.Node node, StreamTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator; streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); return typedTranslator.canTranslate(typedTransform, streamingContext); }
private <T extends PTransform<?, ?>> void applyStreamingTransform( PTransform<?, ?> transform, TransformHierarchy.Node node, StreamTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator; // create the applied PTransform on the streamingContext streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); typedTranslator.translateNode(typedTransform, streamingContext); }
/** * @param beamNode the beam node to be translated. * @param pipeline pipeline. * @return true if the main input has global window. */ private static boolean isGlobalWindow(final TransformHierarchy.Node beamNode, final Pipeline pipeline) { final AppliedPTransform pTransform = beamNode.toAppliedPTransform(pipeline); final PCollection<?> mainInput = (PCollection<?>) Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(pTransform)); return mainInput.getWindowingStrategy().getWindowFn() instanceof GlobalWindows; }
@Override public void visitPrimitiveTransform(Node node) { // TODO: Include DisplayData in the proto children.put(node.getEnclosingNode(), node.toAppliedPTransform(pipeline)); try { components.registerPTransform( node.toAppliedPTransform(pipeline), Collections.emptyList()); } catch (IOException e) { throw new IllegalStateException(e); } } });
@Override public AppliedPTransform<?, ?, ?> getCurrentParent() { if (parents.isEmpty()) { return null; } else { return parents.peekFirst().toAppliedPTransform(getPipeline()); } } }
private <T extends PTransform<?, ?>> void applyBatchTransform( PTransform<?, ?> transform, TransformHierarchy.Node node, BatchTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator; // create the applied PTransform on the batchContext batchContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); typedTranslator.translateNode(typedTransform, batchContext); }
private <T extends PTransform<?, ?>> void applyBatchTransform( PTransform<?, ?> transform, TransformHierarchy.Node node, BatchTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") BatchTransformTranslator<T> typedTranslator = (BatchTransformTranslator<T>) translator; // create the applied PTransform on the batchContext batchContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); typedTranslator.translateNode(typedTransform, batchContext); }
private <T extends PTransform<?, ?>> boolean applyCanTranslate( PTransform<?, ?> transform, TransformHierarchy.Node node, StreamTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator; streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); return typedTranslator.canTranslate(typedTransform, streamingContext); }
private <T extends PTransform<?, ?>> void applyStreamingTransform( PTransform<?, ?> transform, TransformHierarchy.Node node, StreamTransformTranslator<?> translator) { @SuppressWarnings("unchecked") T typedTransform = (T) transform; @SuppressWarnings("unchecked") StreamTransformTranslator<T> typedTranslator = (StreamTransformTranslator<T>) translator; // create the applied PTransform on the streamingContext streamingContext.setCurrentTransform(node.toAppliedPTransform(getPipeline())); typedTranslator.translateNode(typedTransform, streamingContext); }
private void checkForMatches(Node node) { for (PTransformOverride override : overrides) { if (override .getMatcher() .matchesDuringValidation(node.toAppliedPTransform(getPipeline()))) { matched.put(node, override); } } } });