/** Returns the standard URN of a given enum annotated with [(standard_urn)]. */ public static String getUrn(ProtocolMessageEnum value) { return value.getValueDescriptor().getOptions().getExtension(RunnerApi.beamUrn); } }
/** Parses the max timestamp for global windows from the proto. */ private static Instant extractMaxTimestampFromProto() { return new Instant( Long.parseLong( RunnerApi.BeamConstants.Constants.GLOBAL_WINDOW_MAX_TIMESTAMP_MILLIS .getValueDescriptor() .getOptions() .getExtension(RunnerApi.beamConstant))); } }
/** Converts the provided Json{@link String} into {@link PipelineOptions}. */ public static PipelineOptions fromJson(String optionsJson) { try { Map<String, Object> probingOptionsMap = MAPPER.readValue(optionsJson, new TypeReference<Map<String, Object>>() {}); if (probingOptionsMap.containsKey("options")) { //Legacy options. return MAPPER.readValue(optionsJson, PipelineOptions.class); } else { // Fn Options with namespace and version. Struct.Builder builder = Struct.newBuilder(); JsonFormat.parser().merge(optionsJson, builder); return fromProto(builder.build()); } } catch (IOException e) { throw new RuntimeException("Failed to read PipelineOptions from JSON", e); } }
private static SdkFunctionSpec toProto(String urn, Serializable serializable) { return SdkFunctionSpec.newBuilder() .setSpec( FunctionSpec.newBuilder() .setUrn(urn) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(serializable))) .build()) .build(); }
@Test public void structWithNullOptionsDeserializes() throws Exception { Struct serialized = Struct.newBuilder() .putFields( "beam:option:option_key:v1", Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) .build(); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized, notNullValue()); } }
@Override public void onNext(ArtifactChunk value) { try { target.write(value.getData().toByteArray()); } catch (IOException e) { // This should never happen throw new AssertionError(e); } }
public static RunnerApi.DisplayData toProto(DisplayData displayData) { // TODO https://issues.apache.org/jira/browse/BEAM-2645 return RunnerApi.DisplayData.newBuilder() .addItems( RunnerApi.DisplayData.Item.newBuilder() .setId(RunnerApi.DisplayData.Identifier.newBuilder().setKey("stubImplementation")) .setLabel("Stub implementation") .setType(RunnerApi.DisplayData.Type.Enum.BOOLEAN) .setValue(Any.pack(BoolValue.newBuilder().setValue(true).build()))) .build(); } }
@Test public void emptyStructDeserializes() throws Exception { Struct serialized = Struct.getDefaultInstance(); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized, notNullValue()); }
private static Environment createEmbeddedEnvironment(String config) { return Environment.newBuilder() .setUrn(ENVIRONMENT_EMBEDDED) .setPayload(ByteString.copyFromUtf8(MoreObjects.firstNonNull(config, ""))) .build(); }
@Test public void testToFromProto() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct); Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct); assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap())); }
@Override public FunctionSpec translate( AppliedPTransform<?, ?, View.CreatePCollectionView<?, ?>> transform, SdkComponents components) { return FunctionSpec.newBuilder() .setUrn(getUrn(transform.getTransform())) .setPayload( ByteString.copyFrom( SerializableUtils.serializeToByteArray(transform.getTransform().getView()))) .build(); } }
private static Coder<?> fromCustomCoder(RunnerApi.Coder protoCoder) throws IOException { return (Coder<?>) SerializableUtils.deserializeFromByteArray( protoCoder.getSpec().getSpec().getPayload().toByteArray(), "Custom Coder Bytes"); } }
/** Parses a timestamp from the proto. */ private static Instant extractTimestampFromProto(RunnerApi.BeamConstants.Constants constant) { return new Instant( Long.parseLong( constant.getValueDescriptor().getOptions().getExtension(RunnerApi.beamConstant))); } }
public static SdkFunctionSpec toProto( GlobalCombineFn<?, ?, ?> combineFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_COMBINE_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(combineFn))) .build()) .build(); } }
@Override public void onNext(PutArtifactRequest value) { try { stream.write(value.getData().getData().toByteArray()); } catch (IOException e) { throw new RuntimeException(e); } }
public static SdkFunctionSpec translateWindowMappingFn( WindowMappingFn<?> windowMappingFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_WINDOW_MAPPING_FN_URN) .setPayload( ByteString.copyFrom(SerializableUtils.serializeToByteArray(windowMappingFn))) .build()) .build(); }
public static DoFnAndMainOutput doFnAndMainOutputTagFromProto(SdkFunctionSpec fnSpec) { checkArgument( fnSpec.getSpec().getUrn().equals(CUSTOM_JAVA_DO_FN_URN), "Expected %s to be %s with URN %s, but URN was %s", DoFn.class.getSimpleName(), FunctionSpec.class.getSimpleName(), CUSTOM_JAVA_DO_FN_URN, fnSpec.getSpec().getUrn()); byte[] serializedFn = fnSpec.getSpec().getPayload().toByteArray(); return (DoFnAndMainOutput) SerializableUtils.deserializeFromByteArray(serializedFn, "Custom DoFn And Main Output tag"); }
public static SdkFunctionSpec translateViewFn(ViewFn<?, ?> viewFn, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(CUSTOM_JAVA_VIEW_FN_URN) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(viewFn))) .build()) .build(); }
public static UnboundedSource<?, ?> unboundedSourceFromProto(ReadPayload payload) throws InvalidProtocolBufferException { checkArgument(payload.getIsBounded().equals(IsBounded.Enum.UNBOUNDED)); return (UnboundedSource<?, ?>) SerializableUtils.deserializeFromByteArray( payload.getSource().getSpec().getPayload().toByteArray(), "UnboundedSource"); }
private static SdkFunctionSpec toProto(BoundedSource<?> source, SdkComponents components) { return SdkFunctionSpec.newBuilder() .setEnvironmentId(components.getOnlyEnvironmentId()) .setSpec( FunctionSpec.newBuilder() .setUrn(JAVA_SERIALIZED_BOUNDED_SOURCE) .setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(source))) .build()) .build(); }