@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
/** * Defines names of counters we track for each operation. * * There are two types of counters: -- METHOD_NAME : Number of successful invocations of method * METHOD. -- METHOD_NAME_TIME : Total inclusive time spent in method METHOD. */ public enum Counter { BYTES_WRITTEN, CLOSE_CALLS, CLOSE_TOTAL_TIME, JOBS_INSERTED, WRITE_CALLS, WRITE_TOTAL_TIME, }
TableSchema schema = new TableSchema(); schema.setFields(new ArrayList<TableFieldSchema>()); JacksonFactory JACKSON = new JacksonFactory(); JACKSON.createJsonParser(new FileInputStream("schema.json")) .parseArrayAndClose(schema.getFields(), TableFieldSchema.class, null); schema.setFactory(JACKSON); TableReference destTable = new TableReference(); destTable.setProjectId(projectId); destTable.setDatasetId(datasetId); destTable.setTableId(tableId); FileContent content = new FileContent("application/octet-stream", new File(csv)); Job job = new Job(); JobConfiguration config = new JobConfiguration(); JobConfigurationLoad configLoad = new JobConfigurationLoad(); configLoad.setSchema(schema); configLoad.setDestinationTable(destTable); configLoad.setEncoding("UTF-8"); configLoad.setCreateDisposition("CREATE_IF_NEEDED"); config.setLoad(configLoad); job.setConfiguration(config); Insert insert = bigquery.jobs().insert(projectId, job, content); insert.setProjectId(projectId); JobReference jobRef = insert.execute().getJobReference(); String jobId = jobRef.getJobId();
table.setDatasetId(exporterConfig.getBigqueryDatasetId()); table.setTableId(kind + datatableSuffix); loadConfig.setDestinationTable(table);
JobConfigurationLoad loadConfig = new JobConfigurationLoad() .setDestinationTable(ref) .setSchema(schema) .setSourceUris(gcsUris)
loadConfig.setSourceFormat(sourceFormat.getFormatIdentifier()); loadConfig.setSourceUris(gcsPaths); loadConfig.setDestinationTable(tableRef); loadConfig.setWriteDisposition(writeDisposition); if (!Strings.isNullOrEmpty(kmsKeyName)) {
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
@Override com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString());
com.google.api.services.bigquery.model.JobConfiguration toPb() { JobConfigurationLoad loadConfigurationPb = new JobConfigurationLoad(); loadConfigurationPb.setDestinationTable(destinationTable.toPb()); if (createDisposition != null) { loadConfigurationPb.setCreateDisposition(createDisposition.toString()); } if (writeDisposition != null) { loadConfigurationPb.setWriteDisposition(writeDisposition.toString()); } if (csvOptions() != null) { CsvOptions csvOptions = csvOptions(); loadConfigurationPb.setFieldDelimiter(csvOptions.fieldDelimiter()) .setAllowJaggedRows(csvOptions.allowJaggedRows()) .setAllowQuotedNewlines(csvOptions.allowQuotedNewLines()) .setEncoding(csvOptions.encoding()) .setQuote(csvOptions.quote()) .setSkipLeadingRows(csvOptions.skipLeadingRows()); } if (schema != null) { loadConfigurationPb.setSchema(schema.toPb()); } if (formatOptions != null) { loadConfigurationPb.setSourceFormat(formatOptions.type()); } loadConfigurationPb.setMaxBadRecords(maxBadRecords); loadConfigurationPb.setIgnoreUnknownValues(ignoreUnknownValues); loadConfigurationPb.setProjectionFields(projectionFields); return new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(loadConfigurationPb); }
@Override protected JobConfiguration jobConfiguration(String projectId) { JobConfigurationLoad cfg = new JobConfigurationLoad() .setSourceUris(sourceUris(params)); if (params.has("schema")) { cfg.setSchema(tableSchema(params)); } Optional<DatasetReference> defaultDataset = params.getOptional("dataset", String.class) .transform(Bq::datasetReference); String destinationTable = params.get("destination_table", String.class); cfg.setDestinationTable(tableReference(projectId, defaultDataset, destinationTable)); params.getOptional("create_disposition", String.class).transform(cfg::setCreateDisposition); params.getOptional("write_disposition", String.class).transform(cfg::setWriteDisposition); params.getOptional("source_format", String.class).transform(cfg::setSourceFormat); params.getOptional("field_delimiter", String.class).transform(cfg::setFieldDelimiter); params.getOptional("skip_leading_rows", int.class).transform(cfg::setSkipLeadingRows); params.getOptional("encoding", String.class).transform(cfg::setEncoding); params.getOptional("quote", String.class).transform(cfg::setQuote); params.getOptional("max_bad_records", int.class).transform(cfg::setMaxBadRecords); params.getOptional("allow_quoted_newlines", boolean.class).transform(cfg::setAllowQuotedNewlines); params.getOptional("allow_jagged_rows", boolean.class).transform(cfg::setAllowJaggedRows); params.getOptional("ignore_unknown_values", boolean.class).transform(cfg::setIgnoreUnknownValues); params.getOptional("projection_fields", new TypeReference<List<String>>() {}).transform(cfg::setProjectionFields); params.getOptional("autodetect", boolean.class).transform(cfg::setAutodetect); params.getOptional("schema_update_options", new TypeReference<List<String>>() {}).transform(cfg::setSchemaUpdateOptions); return new JobConfiguration() .setLoad(cfg); }