@VisibleForTesting /** This method is for test usage only */ public Write<T> withTestServices(BigQueryServices testServices) { checkArgument(testServices != null, "testServices can not be null"); return toBuilder().setBigQueryServices(testServices).build(); }
@VisibleForTesting Write<T> withMaxFileSize(long maxFileSize) { checkArgument(maxFileSize > 0, "maxFileSize must be > 0, but was: %s", maxFileSize); return toBuilder().setMaxFileSize(maxFileSize).build(); }
@VisibleForTesting Write<T> withMaxFilesPerBundle(int maxFilesPerBundle) { checkArgument( maxFilesPerBundle > 0, "maxFilesPerBundle must be > 0, but was: %s", maxFilesPerBundle); return toBuilder().setMaxFilesPerBundle(maxFilesPerBundle).build(); }
/** * Insert all valid rows of a request, even if invalid rows exist. This is only applicable when * the write method is set to {@link Method#STREAMING_INSERTS}. The default value is false, * which causes the entire request to fail if any invalid rows exist. */ public Write<T> skipInvalidRows() { return toBuilder().setSkipInvalidRows(true).build(); }
/** * Allows the schemas for each table to be computed within the pipeline itself. * * <p>The input is a map-valued {@link PCollectionView} mapping string tablespecs to * JSON-formatted {@link TableSchema}s. Tablespecs must be in the same format as taken by {@link * #to(String)}. */ public Write<T> withSchemaFromView(PCollectionView<Map<String, String>> view) { checkArgument(view != null, "view can not be null"); return toBuilder().setSchemaFromView(view).build(); }
/** Writes to the table and schema specified by the {@link DynamicDestinations} object. */ public Write<T> to(DynamicDestinations<T, ?> dynamicDestinations) { checkArgument(dynamicDestinations != null, "dynamicDestinations can not be null"); return toBuilder().setDynamicDestinations(dynamicDestinations).build(); }
/** Specifies the table description. */ public Write<T> withTableDescription(String tableDescription) { checkArgument(tableDescription != null, "tableDescription can not be null"); return toBuilder().setTableDescription(tableDescription).build(); }
/** The same as {@link #withTimePartitioning}, but takes a JSON-serialized object. */ public Write<T> withJsonTimePartitioning(ValueProvider<String> partitioning) { checkArgument(partitioning != null, "partitioning can not be null"); return toBuilder().setJsonTimePartitioning(partitioning).build(); }
/** Formats the user's type into a {@link TableRow} to be written to BigQuery. */ public Write<T> withFormatFunction(SerializableFunction<T, TableRow> formatFunction) { checkArgument(formatFunction != null, "formatFunction can not be null"); return toBuilder().setFormatFunction(formatFunction).build(); }
/** * Writes to table specified by the specified table function. The table is a function of {@link * ValueInSingleWindow}, so can be determined by the value or by the window. */ public Write<T> to( SerializableFunction<ValueInSingleWindow<T>, TableDestination> tableFunction) { checkArgument(tableFunction != null, "tableFunction can not be null"); return toBuilder().setTableFunction(tableFunction).build(); }
/** Disables BigQuery table validation. */ public Write<T> withoutValidation() { return toBuilder().setValidate(false).build(); }
public Write<T> withLoadJobProjectId(ValueProvider<String> loadJobProjectId) { checkArgument(loadJobProjectId != null, "loadJobProjectId can not be null"); return toBuilder().setLoadJobProjectId(loadJobProjectId).build(); }
/** * Accept rows that contain values that do not match the schema. The unknown values are ignored. * Default is false, which treats unknown values as errors. */ public Write<T> ignoreUnknownValues() { return toBuilder().setIgnoreUnknownValues(true).build(); }
/** * Enables extended error information by enabling {@link WriteResult#getFailedInsertsWithErr()} * * <p>ATM this only works if using {@link Method#STREAMING_INSERTS}. See {@link * Write#withMethod(Method)}. */ public Write<T> withExtendedErrorInfo() { return toBuilder().setExtendedErrorInfo(true).build(); }
/** * Control how many file shards are written when using BigQuery load jobs. Applicable only when * also setting {@link #withTriggeringFrequency}. The default value is 1000. */ @Experimental public Write<T> withNumFileShards(int numFileShards) { checkArgument(numFileShards > 0, "numFileShards must be > 0, but was: %s", numFileShards); return toBuilder().setNumFileShards(numFileShards).build(); }
/** Specifies what to do with existing data in the table, in case the table already exists. */ public Write<T> withWriteDisposition(WriteDisposition writeDisposition) { checkArgument(writeDisposition != null, "writeDisposition can not be null"); return toBuilder().setWriteDisposition(writeDisposition).build(); }
/** Same as {@link #to(String)}, but with a {@link ValueProvider}. */ public Write<T> to(ValueProvider<String> tableSpec) { checkArgument(tableSpec != null, "tableSpec can not be null"); return toBuilder() .setJsonTableRef( NestedValueProvider.of( NestedValueProvider.of(tableSpec, new TableSpecToTableRef()), new TableRefToJson())) .build(); }
/** * Provides a custom location on GCS for storing temporary files to be loaded via BigQuery batch * load jobs. See "Usage with templates" in {@link BigQueryIO} documentation for discussion. */ public Write<T> withCustomGcsTempLocation(ValueProvider<String> customGcsTempLocation) { checkArgument(customGcsTempLocation != null, "customGcsTempLocation can not be null"); return toBuilder().setCustomGcsTempLocation(customGcsTempLocation).build(); }
/** Specifies whether the table should be created if it does not exist. */ public Write<T> withCreateDisposition(CreateDisposition createDisposition) { checkArgument(createDisposition != null, "createDisposition can not be null"); return toBuilder().setCreateDisposition(createDisposition).build(); }
/** * Specfies a policy for handling fPailed inserts. * * <p>Currently this only is allowed when writing an unbounded collection to BigQuery. Bounded * collections are written using batch load jobs, so we don't get per-element failures. * Unbounded collections are written using streaming inserts, so we have access to per-element * insert results. */ public Write<T> withFailedInsertRetryPolicy(InsertRetryPolicy retryPolicy) { checkArgument(retryPolicy != null, "retryPolicy can not be null"); return toBuilder().setFailedInsertRetryPolicy(retryPolicy).build(); }