/** The default {@link PipelineOptions} which will be used by {@link #create()}. */ public static PipelineOptions getDefaultOptions() { return TestPipeline.testingPipelineOptions(); }
/** * Creates and returns a new test pipeline. * * <p>Use {@link PAssert} to add tests, then call {@link Pipeline#run} to execute the pipeline and * check the tests. */ public static TestPipeline create() { return fromOptions(testingPipelineOptions()); }
/** * Creates an instance of this rule. * * <p>Loads GCP configuration from {@link TestPipelineOptions}. */ public static TestPubsubSignal create() { TestPubsubOptions options = TestPipeline.testingPipelineOptions().as(TestPubsubOptions.class); return new TestPubsubSignal(options); }
/** * Creates an instance of this rule. * * <p>Loads GCP configuration from {@link TestPipelineOptions}. */ public static TestBigQuery create(Schema tableSchema) { return new TestBigQuery( TestPipeline.testingPipelineOptions().as(TestBigQueryOptions.class), tableSchema); }
/** * Creates an instance of this rule. * * <p>Loads GCP configuration from {@link TestPipelineOptions}. */ public static TestPubsub create() { TestPubsubOptions options = TestPipeline.testingPipelineOptions().as(TestPubsubOptions.class); return new TestPubsub(options); }
@Test public void testCreationOfPipelineOptions() throws Exception { String stringOptions = MAPPER.writeValueAsString( new String[] {"--runner=org.apache.beam.sdk.testing.CrashingRunner"}); System.getProperties().put("beamTestPipelineOptions", stringOptions); PipelineOptions options = TestPipeline.testingPipelineOptions(); assertEquals(CrashingRunner.class, options.getRunner()); }
@Before public void setup() { PipelineOptionsFactory.register(V1TestOptions.class); options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); ancestor = UUID.randomUUID().toString(); }
@Test public void testStreamingWriteWithNoShardingReturnsNewTransformMaxWorkersUnset() { PipelineOptions options = TestPipeline.testingPipelineOptions(); testStreamingWriteOverride(options, StreamingShardedWriteFactory.DEFAULT_NUM_SHARDS); }
@Before public void setup() { context = mock(EvaluationContext.class); runner = DirectRunner.fromOptions(TestPipeline.testingPipelineOptions()); factory = new TestStreamEvaluatorFactory(context); bundleFactory = ImmutableListBundleFactory.create(); }
@BeforeClass public static void setUp() { PipelineOptionsFactory.register(HIFITestOptions.class); options = TestPipeline.testingPipelineOptions().as(HIFITestOptions.class); }
public static <T extends IOTestPipelineOptions> T readIOTestPipelineOptions( Class<T> optionsType) { PipelineOptionsFactory.register(optionsType); IOTestPipelineOptions options = TestPipeline.testingPipelineOptions().as(optionsType); return PipelineOptionsValidator.validate(optionsType, options); }
@BeforeClass public static void setUp() { PipelineOptionsFactory.register(HIFITestOptions.class); options = TestPipeline.testingPipelineOptions().as(HIFITestOptions.class); }
private void setupTestEnvironment(String recordSize, boolean enableCustomBigquery) { PipelineOptionsFactory.register(BigQueryIOReadOptions.class); options = TestPipeline.testingPipelineOptions().as(BigQueryIOReadOptions.class); options.setNumRecords(numOfRecords.get(recordSize)); options.setTempLocation(options.getTempRoot() + "/temp-it/"); project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); options.setInputTable(project + ":" + datasetId + "." + tablePrefix + recordSize); if (enableCustomBigquery) { options.setExperiments( ImmutableList.of("enable_custom_bigquery_sink", "enable_custom_bigquery_source")); } }
@Before public void setupTestEnvironment() { PipelineOptionsFactory.register(TrafficRoutesOptions.class); this.options = TestPipeline.testingPipelineOptions().as(TrafficRoutesOptions.class); this.projectId = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); this.bqClient = new BigqueryClient("TrafficRoutesIT"); this.bqClient.createNewDataset(this.projectId, this.outputDatasetId); }
@Before public void setupTestEnvironment() { PipelineOptionsFactory.register(TrafficMaxLaneFlowOptions.class); this.options = TestPipeline.testingPipelineOptions().as(TrafficMaxLaneFlowOptions.class); this.projectId = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject(); this.bqClient = new BigqueryClient("TrafficMaxLaneFlowIT"); this.bqClient.createNewDataset(this.projectId, this.outputDatasetId); }
@Test public void testStreamingWriteWithNoShardingReturnsNewTransform() { PipelineOptions options = TestPipeline.testingPipelineOptions(); options.as(DataflowPipelineWorkerPoolOptions.class).setMaxNumWorkers(10); testStreamingWriteOverride(options, 20); }
@Test public void testNestedRecords() throws Exception { PipelineOptionsFactory.register(Options.class); TestPipelineOptions testOptions = TestPipeline.testingPipelineOptions().as(TestPipelineOptions.class); Options options = testOptions.as(Options.class); options.setTempLocation(testOptions.getTempRoot() + "/temp-it/"); runPipeline(options); }
@Test public void testWriteWithoutValidate() { final String table = "fooTable"; BigtableIO.Write write = BigtableIO.write() .withBigtableOptions(BIGTABLE_OPTIONS) .withTableId(table) .withBigtableService(service) .withoutValidation(); // validate() will throw if withoutValidation() isn't working write.validate(TestPipeline.testingPipelineOptions()); }
@Test public void testE2EAutoComplete() throws Exception { AutoCompleteITOptions options = TestPipeline.testingPipelineOptions().as(AutoCompleteITOptions.class); options.setInputFile(DEFAULT_INPUT); options.setOutputToBigQuery(false); options.setOutputToDatastore(false); options.setOutputToChecksum(true); options.setExpectedChecksum(DEFAULT_INPUT_CHECKSUM); AutoComplete.runAutocompletePipeline(options); } }
@Test public void testPipelineUserExceptionHandling() { PipelineOptions options = TestPipeline.testingPipelineOptions(); options.setRunner(TestPipelineRunnerThrowingUserException.class); Pipeline p = Pipeline.create(options); // Check pipeline runner correctly catches user errors. thrown.expect(PipelineExecutionException.class); thrown.expectCause(isA(IllegalStateException.class)); thrown.expectMessage("user code exception"); p.run(); }