/** * Once set, all {@code RuntimeValueProviders} will return {@code true} from {@code * isAccessible()}. By default, the value is set when deserializing {@link PipelineOptions}. */ static void setRuntimeOptions(PipelineOptions runtimeOptions) { optionsMap.put(runtimeOptions.getOptionsId(), runtimeOptions); }
@StartBundle public void startBundle(StartBundleContext context) { long id = context.getPipelineOptions().getOptionsId(); values = globalValues.get(id); }
propertyName, (Class<? extends PipelineOptions>) method.getDeclaringClass(), proxy.getOptionsId()) : new RuntimeValueProvider( method.getName(), (Class<? extends PipelineOptions>) method.getDeclaringClass(), defaultObject, proxy.getOptionsId()); } else if (defaultObject != null) { return defaultObject;
private static Enumerable<Object> limitCollect(PipelineOptions options, BeamRelNode node) { long id = options.getOptionsId(); Queue<Object> values = new ConcurrentLinkedQueue<>(); checkArgument( options .getRunner() .getCanonicalName() .equals("org.apache.beam.runners.direct.DirectRunner"), "SELECT without INSERT is only supported in DirectRunner in SQL Shell."); int limitCount = getLimitCount(node); Collector.globalValues.put(id, values); limitRun(options, node, new Collector(), values, limitCount); Collector.globalValues.remove(id); // remove extra retrieved values while (values.size() > limitCount) { values.remove(); } return Linq4j.asEnumerable(values); }
@Test public void testPipelineOptionsIdIsUniquePerInstance() { Set<Long> ids = new HashSet<>(); for (int i = 0; i < 1000; ++i) { long id = PipelineOptionsFactory.create().getOptionsId(); if (!ids.add(id)) { fail(String.format("Generated duplicate id %s, existing generated ids %s", id, ids)); } } }
private static Enumerable<Object> collect(PipelineOptions options, BeamRelNode node) { long id = options.getOptionsId(); Queue<Object> values = new ConcurrentLinkedQueue<>(); checkArgument( options .getRunner() .getCanonicalName() .equals("org.apache.beam.runners.direct.DirectRunner"), "SELECT without INSERT is only supported in DirectRunner in SQL Shell."); Collector.globalValues.put(id, values); Pipeline pipeline = Pipeline.create(options); PCollection<Row> resultCollection = BeamSqlRelUtils.toPCollection(pipeline, node); resultCollection.apply(ParDo.of(new Collector())); PipelineResult result = pipeline.run(); result.waitUntilFinish(); Collector.globalValues.remove(id); return Linq4j.asEnumerable(values); }
@Test public void testOptionsIdIsSet() throws Exception { ObjectMapper mapper = new ObjectMapper() .registerModules(ObjectMapper.findModules(ReflectHelpers.findClassLoader())); PipelineOptions options = PipelineOptionsFactory.create(); // We purposely serialize/deserialize to get another instance. This allows to test if the // default has been set or not. PipelineOptions clone = mapper.readValue(mapper.writeValueAsString(options), PipelineOptions.class); // It is important that we don't call getOptionsId() before we have created the clone. assertEquals(options.getOptionsId(), clone.getOptionsId()); }
@Test public void testToFromProto() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct); Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct); assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap())); }
@Test public void testToFromJson() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); String json = PipelineOptionsTranslation.toJson(options); String legacyJson = MAPPER.writeValueAsString(options); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(json)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(legacyJson)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); } }