private StreamingWriteTables( BigQueryServices bigQueryServices, InsertRetryPolicy retryPolicy, boolean extendedErrorInfo, boolean skipInvalidRows, boolean ignoreUnknownValues) { this.bigQueryServices = bigQueryServices; this.retryPolicy = retryPolicy; this.extendedErrorInfo = extendedErrorInfo; this.skipInvalidRows = skipInvalidRows; this.ignoreUnknownValues = ignoreUnknownValues; }
@Test public void testDontRetryPersistentErrors() { InsertRetryPolicy policy = InsertRetryPolicy.retryTransientErrors(); assertTrue( policy.shouldRetry(new Context(generateErrorAmongMany(5, "timeout", "unavailable")))); assertFalse(policy.shouldRetry(new Context(generateErrorAmongMany(5, "timeout", "invalid")))); assertFalse( policy.shouldRetry(new Context(generateErrorAmongMany(5, "timeout", "invalidQuery")))); assertFalse( policy.shouldRetry(new Context(generateErrorAmongMany(5, "timeout", "notImplemented")))); }
@Test public void testNeverRetry() { assertFalse( InsertRetryPolicy.neverRetry() .shouldRetry(new Context(new TableDataInsertAllResponse.InsertErrors()))); }
@Test public void testAlwaysRetry() { assertTrue( InsertRetryPolicy.alwaysRetry() .shouldRetry(new Context(new TableDataInsertAllResponse.InsertErrors()))); }
BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.retryTransientErrors(), failedInserts, ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
if (retryPolicy.shouldRetry(new InsertRetryPolicy.Context(error))) { allErrors.add(error); retryRows.add(rowsToPublish.get(errorIndex));
BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.neverRetry(), failedInserts, ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation();
if (allErrors != null) { for (TableDataInsertAllResponse.InsertErrors errors : allErrors) { if (!retryPolicy.shouldRetry(new Context(errors))) { shouldInsert = false;
BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.neverRetry(), failedInserts, ErrorContainer.BIG_QUERY_INSERT_ERROR_ERROR_CONTAINER,
/** Constructor. */ public StreamingInserts( CreateDisposition createDisposition, DynamicDestinations<?, DestinationT> dynamicDestinations) { this( createDisposition, dynamicDestinations, new BigQueryServicesImpl(), InsertRetryPolicy.alwaysRetry(), false, false, false); }
new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation())
BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.neverRetry(), Lists.newArrayList(), ErrorContainer.TABLE_ROW_ERROR_CONTAINER, BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.neverRetry(), Lists.newArrayList(), ErrorContainer.TABLE_ROW_ERROR_CONTAINER,
public long insertAll(TableReference ref, List<TableRow> rowList) throws IOException, InterruptedException { List<ValueInSingleWindow<TableRow>> rows = rowList.stream() .map(r -> ValueInSingleWindow.of( r, BoundedWindow.TIMESTAMP_MIN_VALUE, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING)) .collect(Collectors.toList()); return bqServices.getDatasetService(bqOptions) .insertAll(ref, rows, null, InsertRetryPolicy.alwaysRetry(), new ArrayList<>(), ErrorContainer.TABLE_ROW_ERROR_CONTAINER, false, false); }
new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withTestServices(fakeBqServices) .withoutValidation()
public long insertAll( TableReference ref, List<TableRow> rowList, @Nullable List<String> insertIdList) throws IOException, InterruptedException { List<ValueInSingleWindow<TableRow>> windowedRows = Lists.newArrayList(); for (TableRow row : rowList) { windowedRows.add( ValueInSingleWindow.of( row, GlobalWindow.TIMESTAMP_MAX_VALUE, GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING)); } return insertAll( ref, windowedRows, insertIdList, InsertRetryPolicy.alwaysRetry(), null, null, false, false); }
.withWriteDisposition(WriteDisposition.WRITE_APPEND) .withMethod(Method.STREAMING_INSERTS) .withFailedInsertRetryPolicy(InsertRetryPolicy.retryTransientErrors()) .withCustomGcsTempLocation(options.getBigQueryLoadingTemporaryDirectory()));
totalBytes = datasetService.insertAll( ref, rows, ids, InsertRetryPolicy.alwaysRetry(), null, null, false, false); } finally { verifyInsertAll(5);
BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.alwaysRetry(), null, null,
BackOffAdapter.toGcpBackOff(TEST_BACKOFF.backoff()), new MockSleeper(), InsertRetryPolicy.alwaysRetry(), null, null,