/** * Gets the table specified by the configuration. * @return The table ID. */ public String getTableId() { return tableId.get(); }
@Override public T apply(Void input) { return provider.get(); } }))
/** * Gets the project ID for the Cloud Bigtable instance. * @return The project ID for the instance. */ public String getProjectId() { return configuration.get(BigtableOptionsFactory.PROJECT_ID_KEY).get(); }
@Override public TableSchema getSchema(TableDestination destination) { String jsonSchema = this.jsonSchema.get(); checkArgument(jsonSchema != null, "jsonSchema can not be null"); return BigQueryHelpers.fromJsonString(jsonSchema, TableSchema.class); }
ConstantTimePartitioningDestinations( DynamicDestinations<T, TableDestination> inner, ValueProvider<String> jsonTimePartitioning) { super(inner); checkArgument(jsonTimePartitioning != null, "jsonTimePartitioning provider can not be null"); if (jsonTimePartitioning.isAccessible()) { checkArgument(jsonTimePartitioning.get() != null, "jsonTimePartitioning can not be null"); } this.jsonTimePartitioning = jsonTimePartitioning; }
@Override public void serialize(ValueProvider<?> value, JsonGenerator jgen, SerializerProvider provider) throws IOException { if (value.isAccessible()) { jgen.writeObject(value.get()); } else { jgen.writeNull(); } } }
@ProcessElement public void processElement(ProcessContext c) { KV<String, TableManifest> kv = c.element(); String table = kv.getKey(); TableManifest manifest = kv.getValue(); boolean gcs = GcsPath.GCS_URI.matcher(importDirectory.get()).matches(); if (gcs) { validateGcsFiles(c, table, manifest); } else { validateLocalFiles(c, table, manifest); } }
@Override public void encode(Params value, OutputStream outStream) throws IOException { if (value == null) { throw new CoderException("cannot encode a null value"); } stringCoder.encode(value.baseFilename.get().toString(), outStream); stringCoder.encode(value.shardTemplate, outStream); stringCoder.encode(value.suffix, outStream); }
/** Like {@code topic()} but with a {@link ValueProvider}. */ public Read<T> fromTopic(ValueProvider<String> topic) { if (topic.isAccessible()) { // Validate. PubsubTopic.fromPath(topic.get()); } return toBuilder() .setTopicProvider(NestedValueProvider.of(topic, new TopicTranslator())) .build(); }
@Test public void testSerializeDeserializeWithArg() throws Exception { TestOptions submitOptions = PipelineOptionsFactory.fromArgs("--foo=baz").as(TestOptions.class); assertTrue(submitOptions.getFoo().isAccessible()); assertEquals("baz", submitOptions.getFoo().get()); ObjectNode root = MAPPER.valueToTree(submitOptions); ((ObjectNode) root.get("options")).put("foo", "quux"); TestOptions runtime = MAPPER.convertValue(root, PipelineOptions.class).as(TestOptions.class); ValueProvider<String> vp = runtime.getFoo(); assertTrue(vp.isAccessible()); assertEquals("quux", vp.get()); }
@Test public void testStaticValueProvider() { ValueProvider<String> provider = StaticValueProvider.of("foo"); assertEquals("foo", provider.get()); assertTrue(provider.isAccessible()); assertEquals("foo", provider.toString()); }
@Test public void testBuildWriteWithTableReference() { TableReference table = new TableReference() .setProjectId("foo.com:project") .setDatasetId("somedataset") .setTableId("sometable"); BigQueryIO.Write<TableRow> write = BigQueryIO.writeTableRows().to(table); assertEquals("foo.com:project", write.getTable().get().getProjectId()); assertEquals("somedataset", write.getTable().get().getDatasetId()); assertEquals("sometable", write.getTable().get().getTableId()); }
@Test public void testStringListValueProvider() { String[] args = new String[] {"--stringValue=abc", "--stringValue=xyz"}; String[] commaArgs = new String[] {"--stringValue=abc,xyz"}; String[] emptyArgs = new String[] {"--stringValue=", "--stringValue="}; Lists options = PipelineOptionsFactory.fromArgs(args).as(Lists.class); assertEquals(ImmutableList.of("abc", "xyz"), options.getStringValue().get()); options = PipelineOptionsFactory.fromArgs(commaArgs).as(Lists.class); assertEquals(ImmutableList.of("abc", "xyz"), options.getStringValue().get()); options = PipelineOptionsFactory.fromArgs(emptyArgs).as(Lists.class); assertEquals(ImmutableList.of("", ""), options.getStringValue().get()); }
@Test public void testCommandLineWithDefault() { TestOptions options = PipelineOptionsFactory.fromArgs("--bar=baz").as(TestOptions.class); ValueProvider<String> provider = options.getBar(); assertEquals("baz", provider.get()); assertTrue(provider.isAccessible()); }
@Test public void testListValueProvider() { TestOptions options = PipelineOptionsFactory.fromArgs("--list=1,2,3").as(TestOptions.class); ValueProvider<List<Integer>> provider = options.getList(); assertEquals(ImmutableList.of(1, 2, 3), provider.get()); assertTrue(provider.isAccessible()); }
@Test public void testNoDefaultRuntimeProvider() { TestOptions options = PipelineOptionsFactory.as(TestOptions.class); ValueProvider<String> provider = options.getFoo(); assertFalse(provider.isAccessible()); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Value only available at runtime"); expectedException.expectMessage("foo"); provider.get(); }
@Test public void testDefaultRuntimeProviderWithoutOverride() throws Exception { TestOptions runtime = PipelineOptionsFactory.as(TestOptions.class); TestOptions options = PipelineOptionsFactory.as(TestOptions.class); runtime.setOptionsId(options.getOptionsId()); RuntimeValueProvider.setRuntimeOptions(runtime); ValueProvider<String> provider = options.getBar(); assertTrue(provider.isAccessible()); assertEquals("bar", provider.get()); }
@Test public void testEnumValueProvider() { String[] args = new String[] {"--enumValue=" + TestEnum.Value}; String[] emptyArgs = new String[] {"--enumValue="}; Objects options = PipelineOptionsFactory.fromArgs(args).as(Objects.class); assertEquals(TestEnum.Value, options.getEnumValue().get()); expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage(emptyStringErrorMessage()); PipelineOptionsFactory.fromArgs(emptyArgs).as(Objects.class); }