public static void main(String[] args) throws IOException, IllegalArgumentException { // Create and set your PipelineOptions. PipelineOptionsFactory.register(SampleOptions.class); SampleOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(SampleOptions.class); runAvroToCsv(options); } }
public static void main(String[] args) throws IOException, IllegalArgumentException { // Create and set your PipelineOptions. PipelineOptionsFactory.register(SampleOptions.class); SampleOptions options = PipelineOptionsFactory.fromArgs(args).withValidation() .as(SampleOptions.class); runCsvToAvro(options); } }
/** * Get command line options */ public static Options getOptions(String[] args) { PipelineOptionsFactory.register(Options.class); Options options = PipelineOptionsFactory.fromArgs(args) .withValidation() .as(Options.class); return options; }
@BeforeClass public static void setUp() { PipelineOptionsFactory.register(HIFITestOptions.class); options = TestPipeline.testingPipelineOptions().as(HIFITestOptions.class); }
public static <T extends IOTestPipelineOptions> T readIOTestPipelineOptions( Class<T> optionsType) { PipelineOptionsFactory.register(optionsType); IOTestPipelineOptions options = TestPipeline.testingPipelineOptions().as(optionsType); return PipelineOptionsValidator.validate(optionsType, options); }
@BeforeClass public static void setUp() { PipelineOptionsFactory.register(HIFITestOptions.class); options = TestPipeline.testingPipelineOptions().as(HIFITestOptions.class); }
@Test public void testManualRegistration() { assertFalse(PipelineOptionsFactory.getRegisteredOptions().contains(TestPipelineOptions.class)); PipelineOptionsFactory.register(TestPipelineOptions.class); assertTrue(PipelineOptionsFactory.getRegisteredOptions().contains(TestPipelineOptions.class)); }
public static void main(String[] args) { PipelineOptionsFactory.register(ExportOptions.class); ExportOptions opts = PipelineOptionsFactory .fromArgs(args).withValidation() .as(ExportOptions.class); Pipeline pipeline = buildPipeline(opts); PipelineResult result = pipeline.run(); if (opts.getWait()) { Utils.waitForPipelineToFinish(result); } }
@BeforeClass public static void setup() { PipelineOptionsFactory.register(KinesisTestOptions.class); options = TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class); }
@BeforeClass public static void setUp() { PipelineOptionsFactory.register(HadoopFormatIOTestOptions.class); options = TestPipeline.testingPipelineOptions().as(HadoopFormatIOTestOptions.class); }
@BeforeClass public static void setUp() { PipelineOptionsFactory.register(HadoopFormatIOTestOptions.class); options = TestPipeline.testingPipelineOptions().as(HadoopFormatIOTestOptions.class); }
@Test public void testResetRegistry() { Set<Class<? extends PipelineOptions>> defaultRegistry = new HashSet<>(PipelineOptionsFactory.getRegisteredOptions()); assertThat(defaultRegistry, not(hasItem(FooOptions.class))); PipelineOptionsFactory.register(FooOptions.class); assertThat(PipelineOptionsFactory.getRegisteredOptions(), hasItem(FooOptions.class)); PipelineOptionsFactory.resetCache(); assertEquals(defaultRegistry, PipelineOptionsFactory.getRegisteredOptions()); }
@Before public void setup() { PipelineOptionsFactory.register(V1TestOptions.class); options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); ancestor = UUID.randomUUID().toString(); }
@Before public void setup() throws Exception { PipelineOptionsFactory.register(V1TestOptions.class); options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); ancestor = UUID.randomUUID().toString(); // Create entities and write them to datastore writeEntitiesToDatastore(options, project, ancestor, numEntities); }
private void setupTestEnvironment(String recordSize, boolean enableCustomBigquery) { PipelineOptionsFactory.register(BigQueryIOReadOptions.class); options = TestPipeline.testingPipelineOptions().as(BigQueryIOReadOptions.class); options.setNumRecords(numOfRecords.get(recordSize)); options.setTempLocation(options.getTempRoot() + "/temp-it/"); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); options.setInputTable(project + ":" + datasetId + "." + tablePrefix + recordSize); if (enableCustomBigquery) { options.setExperiments( ImmutableList.of("enable_custom_bigquery_sink", "enable_custom_bigquery_source")); } }
@Test public void testNestedRecords() throws Exception { PipelineOptionsFactory.register(Options.class); TestPipelineOptions testOptions = TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class); Options options = testOptions.as(Options.class); options.setTempLocation(testOptions.getTempRoot() + "/temp-it/"); runPipeline(options); }
@Test public void testDisplayDataDeserializationWithRegistration() throws Exception { PipelineOptionsFactory.register(HasClassOptions.class); HasClassOptions options = PipelineOptionsFactory.as(HasClassOptions.class); options.setClassOption(ProxyInvocationHandlerTest.class); PipelineOptions deserializedOptions = serializeDeserialize(PipelineOptions.class, options); DisplayData displayData = DisplayData.from(deserializedOptions); assertThat(displayData, hasDisplayItem("classOption", ProxyInvocationHandlerTest.class)); }
@Before public void setupTestEnvironment() { PipelineOptionsFactory.register(TrafficRoutesOptions.class); this.options = TestPipeline.testingPipelineOptions().as(TrafficRoutesOptions.class); this.projectId = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); this.bqClient = new BigqueryClient("TrafficRoutesIT"); this.bqClient.createNewDataset(this.projectId, this.outputDatasetId); }
@Before public void setupTestEnvironment() { PipelineOptionsFactory.register(TrafficMaxLaneFlowOptions.class); this.options = TestPipeline.testingPipelineOptions().as(TrafficMaxLaneFlowOptions.class); this.projectId = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); this.bqClient = new BigqueryClient("TrafficMaxLaneFlowIT"); this.bqClient.createNewDataset(this.projectId, this.outputDatasetId); }