stagePayload = RunnerApi.ExecutableStagePayload.parseFrom(transform.getSpec().getPayload()); } catch (IOException e) { throw new RuntimeException(e); String inputPCollectionId = stagePayload.getInput(); final TransformedSideInputs transformedSideInputs; if (stagePayload.getSideInputsCount() > 0) { transformedSideInputs = transformSideInputs(stagePayload, components, context); } else { stagePayload.getUserStatesCount() > 0 || stagePayload.getTimersCount() > 0; Coder keyCoder = null; KeySelector<WindowedValue<InputT>, ?> keySelector = null;
try { stagePayload = RunnerApi.ExecutableStagePayload.parseFrom( transform.getTransform().getSpec().getPayload()); } catch (IOException e) { String inputPCollectionId = stagePayload.getInput(); Coder<WindowedValue<InputT>> windowedInputCoder = instantiateCoder(inputPCollectionId, components); if (stagePayload.getUserStatesCount() > 0 || stagePayload.getTimersCount() > 0) { for (SideInputId sideInputId : stagePayload.getSideInputsList()) { String collectionId = components
Components components = payload.getComponents(); Environment environment = payload.getEnvironment(); payload.getInput(), components.getPcollectionsOrThrow(payload.getInput())); List<SideInputReference> sideInputs = payload .getSideInputsList() .stream() .map(sideInputId -> SideInputReference.fromSideInputId(sideInputId, components)) List<UserStateReference> userStates = payload .getUserStatesList() .stream() .map(userStateId -> UserStateReference.fromUserStateId(userStateId, components)) List<TimerReference> timers = payload .getTimersList() .stream() .map(timerId -> TimerReference.fromTimerId(timerId, components)) List<PTransformNode> transforms = payload .getTransformsList() .stream() .map(id -> PipelineNode.pTransform(id, components.getTransformsOrThrow(id))) List<PCollectionNode> outputs =
try { stagePayload = RunnerApi.ExecutableStagePayload.parseFrom( transform.getTransform().getSpec().getPayload()); } catch (IOException e) { String inputPCollectionId = stagePayload.getInput(); Coder<WindowedValue<InputT>> windowedInputCoder = instantiateCoder(inputPCollectionId, components); if (stagePayload.getUserStatesCount() > 0 || stagePayload.getTimersCount() > 0) { for (SideInputId sideInputId : stagePayload.getSideInputsList()) { String collectionId = components
stagePayload = RunnerApi.ExecutableStagePayload.parseFrom(transform.getSpec().getPayload()); } catch (IOException e) { throw new RuntimeException(e); String inputPCollectionId = stagePayload.getInput(); final TransformedSideInputs transformedSideInputs; if (stagePayload.getSideInputsCount() > 0) { transformedSideInputs = transformSideInputs(stagePayload, components, context); } else { if (stagePayload.getUserStatesCount() > 0 || stagePayload.getTimersCount() > 0) {
ExecutableStagePayload.parseFrom(ptransform.getSpec().getPayload()); assertThat(payload.getTransformsList(), contains("parDo", "window")); ExecutableStage desered = ExecutableStage.fromPayload(payload); assertThat(desered, equalTo(subgraph));
ExecutableStagePayload.parseFrom(stagePTransform.getSpec().getPayload()); assertThat(payload.getTransformsList(), contains("pt")); assertThat(ExecutableStage.fromPayload(payload), equalTo(stage));
ExecutableStagePayload.parseFrom(stagePTransform.getSpec().getPayload()); assertThat(payload.getTransformsList(), contains("pt")); assertThat(ExecutableStage.fromPayload(payload), equalTo(stage));
this.outputMap = outputMap; this.sideInputIds = sideInputIds; this.usesTimers = payload.getTimersCount() > 0; if (usesTimers) {
stagePayload.getSideInputsList()) {
stagePayload.getSideInputsList()) {
this.outputMap = outputMap; this.sideInputIds = sideInputIds; this.usesTimers = payload.getTimersCount() > 0;
this.stage = ExecutableStage.fromPayload( ExecutableStagePayload.parseFrom(transform.getTransform().getSpec().getPayload())); this.outputs = new ArrayList<>();
private static void validateExecutableStage( String id, PTransform transform, Components outerComponents) throws Exception { ExecutableStagePayload payload = ExecutableStagePayload.parseFrom(transform.getSpec().getPayload()); Components components = payload.getComponents(); transform.getInputsMap().values().contains(payload.getInput()), "ExecutableStage %s uses unknown input %s", id, payload.getInput()); !payload.getTransformsList().isEmpty(), "ExecutableStage %s contains no transforms", id); for (String subtransformId : payload.getTransformsList()) { checkArgument( components.containsTransforms(subtransformId), subtransformId); for (String outputId : payload.getOutputsList()) { checkArgument( components.containsPcollections(outputId),
private RemoteStageEvaluator(PTransformNode transform) throws Exception { this.transform = transform; ExecutableStage stage = ExecutableStage.fromPayload( ExecutableStagePayload.parseFrom(transform.getTransform().getSpec().getPayload())); this.outputs = new ArrayList<>(); StageBundleFactory stageFactory = jobFactory.forStage(stage); this.bundle = stageFactory.getBundle( BundleFactoryOutputReceiverFactory.create( bundleFactory, stage.getComponents(), outputs::add), StateRequestHandler.unsupported(), BundleProgressHandler.ignored()); // TODO(BEAM-4680): Add support for timers as inputs to the ULR this.mainInput = Iterables.getOnlyElement(bundle.getInputReceivers().values()); }
/** Extracts an {@link ExecutableStagePayload} from the given transform. */ public static ExecutableStagePayload getExecutableStagePayload( AppliedPTransform<?, ?, ?> appliedTransform) throws IOException { RunnerApi.PTransform transform = PTransformTranslation.toProto( appliedTransform, SdkComponents.create(appliedTransform.getPipeline().getOptions())); checkArgument(ExecutableStage.URN.equals(transform.getSpec().getUrn())); return ExecutableStagePayload.parseFrom(transform.getSpec().getPayload()); } }
public FlinkExecutableStageFunction( RunnerApi.ExecutableStagePayload stagePayload, JobInfo jobInfo, Map<String, Integer> outputMap, FlinkExecutableStageContext.Factory contextFactory, Coder windowCoder) { this.stagePayload = stagePayload; this.jobInfo = jobInfo; this.outputMap = outputMap; this.contextFactory = contextFactory; this.windowCoder = windowCoder; this.stageName = stagePayload.getInput(); }