private Set<String> getLocalSideInputNames(PTransform transform) { if (PAR_DO_TRANSFORM_URN.equals(transform.getSpec().getUrn())) { try { return ParDoPayload.parseFrom(transform.getSpec().getPayload()).getSideInputsMap().keySet(); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } } else { return Collections.emptySet(); } }
private Set<String> getLocalTimerNames(PTransform transform) { if (PAR_DO_TRANSFORM_URN.equals(transform.getSpec().getUrn())) { try { return ParDoPayload.parseFrom(transform.getSpec().getPayload()).getTimerSpecsMap().keySet(); } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } } else { return Collections.emptySet(); } }
private static void urnNotFound( PTransformNode transform, RunnerApi.Pipeline pipeline, BatchTranslationContext context) { throw new IllegalArgumentException( String.format( "Unknown type of URN %s for PTransform with id %s.", transform.getTransform().getSpec().getUrn(), transform.getId())); }
private static void validateAssignWindows(String id, PTransform transform, Components components) throws Exception { WindowIntoPayload.parseFrom(transform.getSpec().getPayload()); }
@Override public FlinkPortablePipelineTranslator.Executor translate( StreamingTranslationContext context, RunnerApi.Pipeline pipeline) { QueryablePipeline p = QueryablePipeline.forTransforms( pipeline.getRootTransformIdsList(), pipeline.getComponents()); for (PipelineNode.PTransformNode transform : p.getTopologicallyOrderedTransforms()) { urnToTransformTranslator .getOrDefault(transform.getTransform().getSpec().getUrn(), this::urnNotFound) .translate(transform.getId(), pipeline, context); } return context; }
public static PCollection.IsBounded sourceIsBounded(AppliedPTransform<?, ?, ?> transform) { try { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); return PCollectionTranslation.fromProto( ReadPayload.parseFrom( PTransformTranslation.toProto(transform, Collections.emptyList(), components) .getSpec() .getPayload()) .getIsBounded()); } catch (IOException e) { throw new RuntimeException("Internal error determining boundedness of Read", e); } }
private static <T> ParDoPayload getParDoPayload(AppliedPTransform<?, ?, ?> transform) throws IOException { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); RunnerApi.PTransform parDoPTransform = PTransformTranslation.toProto(transform, Collections.emptyList(), components); return ParDoPayload.parseFrom(parDoPTransform.getSpec().getPayload()); }
private static <T> ReadPayload getReadPayload( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) throws IOException { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); return ReadPayload.parseFrom( PTransformTranslation.toProto(transform, Collections.emptyList(), components) .getSpec() .getPayload()); }
public RawWriteFiles( RunnerApi.PTransform protoTransform, RehydratedComponents rehydratedComponents) throws IOException { this.rehydratedComponents = rehydratedComponents; this.protoTransform = protoTransform; this.spec = protoTransform.getSpec(); this.payload = RunnerApi.WriteFilesPayload.parseFrom(spec.getPayload()); }
private static void urnNotFound( PTransformNode transform, RunnerApi.Pipeline pipeline, BatchTranslationContext context) { throw new IllegalArgumentException( String.format( "Unknown type of URN %s for PTransform with id %s.", transform.getTransform().getSpec().getUrn(), transform.getId())); }
private static String windowExtractor(PTransform transform) throws InvalidProtocolBufferException { return WindowIntoPayload.parseFrom(transform.getSpec().getPayload()) .getWindowFn() .getEnvironmentId(); }
private static String combineExtractor(PTransform pTransform) throws InvalidProtocolBufferException { return CombinePayload.parseFrom(pTransform.getSpec().getPayload()) .getCombineFn() .getEnvironmentId(); }
private static <T, DestinationT> WriteFilesPayload getWriteFilesPayload( AppliedPTransform< PCollection<T>, WriteFilesResult<DestinationT>, ? extends PTransform<PCollection<T>, WriteFilesResult<DestinationT>>> transform) throws IOException { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); return WriteFilesPayload.parseFrom( PTransformTranslation.toProto(transform, Collections.emptyList(), components) .getSpec() .getPayload()); }
private RunnerApi.Pipeline makeKnownUrnsPrimitives( RunnerApi.Pipeline pipeline, Set<String> knownUrns) { RunnerApi.Pipeline.Builder trimmedPipeline = pipeline.toBuilder(); for (String ptransformId : pipeline.getComponents().getTransformsMap().keySet()) { if (knownUrns.contains( pipeline.getComponents().getTransformsOrThrow(ptransformId).getSpec().getUrn())) { LOG.debug("Removing descendants of known PTransform {}" + ptransformId); removeDescendants(trimmedPipeline, ptransformId); } } return trimmedPipeline.build(); }
private void urnNotFound( String id, RunnerApi.Pipeline pipeline, FlinkStreamingPortablePipelineTranslator.TranslationContext context) { throw new IllegalArgumentException( String.format( "Unknown type of URN %s for PTransform with id %s.", pipeline.getComponents().getTransformsOrThrow(id).getSpec().getUrn(), id)); }
@Override public void translate(StreamingTranslationContext context, RunnerApi.Pipeline pipeline) { QueryablePipeline p = QueryablePipeline.forTransforms( pipeline.getRootTransformIdsList(), pipeline.getComponents()); for (PipelineNode.PTransformNode transform : p.getTopologicallyOrderedTransforms()) { urnToTransformTranslator .getOrDefault(transform.getTransform().getSpec().getUrn(), this::urnNotFound) .translate(transform.getId(), pipeline, context); } }
/** Extracts an {@link ExecutableStagePayload} from the given transform. */ public static ExecutableStagePayload getExecutableStagePayload( AppliedPTransform<?, ?, ?> appliedTransform) throws IOException { RunnerApi.PTransform transform = PTransformTranslation.toProto( appliedTransform, SdkComponents.create(appliedTransform.getPipeline().getOptions())); checkArgument(ExecutableStage.URN.equals(transform.getSpec().getUrn())); return ExecutableStagePayload.parseFrom(transform.getSpec().getPayload()); } }
public static RunnerApi.PCollection getMainInput( RunnerApi.PTransform ptransform, Components components) throws IOException { checkArgument( ptransform.getSpec().getUrn().equals(PAR_DO_TRANSFORM_URN), "Unexpected payload type %s", ptransform.getSpec().getUrn()); return components.getPcollectionsOrThrow( ptransform.getInputsOrThrow(getMainInputName(ptransform))); }
private static void validateCombine(String id, PTransform transform, Components components) throws Exception { CombinePayload payload = CombinePayload.parseFrom(transform.getSpec().getPayload()); checkArgument( components.containsCoders(payload.getAccumulatorCoderId()), "Transform %s uses unknown accumulator coder id %s", payload.getAccumulatorCoderId()); }