/** Example of creating a channel with which to write to a table. */ // [TARGET writer(JobId, WriteChannelConfiguration)] // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE "StringValue1\nStringValue2\n"] // [VARIABLE "us"] public long writeToTableLocation( String datasetName, String tableName, String csvData, String location) throws IOException, InterruptedException, TimeoutException { // [START ] TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build(); // The location must be specified; other fields can be auto-detected. JobId jobId = JobId.newBuilder().setLocation(location).build(); TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); // Write data to writer try { writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); } finally { writer.close(); } // Get load job Job job = writer.getJob(); job = job.waitFor(); LoadStatistics stats = job.getStatistics(); return stats.getOutputRows(); // [END ] }
@Override public final boolean equals(Object obj) { return obj == this || obj != null && obj.getClass().equals(LoadStatistics.class) && baseEquals((LoadStatistics) obj); }
@Override public final int hashCode() { return Objects.hash( baseHashCode(), inputBytes, inputFiles, outputBytes, outputRows, badRecords); }
@SuppressWarnings("unchecked") static <T extends JobStatistics> T fromPb(com.google.api.services.bigquery.model.Job jobPb) { JobConfiguration jobConfigPb = jobPb.getConfiguration(); com.google.api.services.bigquery.model.JobStatistics statisticPb = jobPb.getStatistics(); if (jobConfigPb.getLoad() != null) { return (T) LoadStatistics.fromPb(statisticPb); } else if (jobConfigPb.getExtract() != null) { return (T) ExtractStatistics.fromPb(statisticPb); } else if (jobConfigPb.getQuery() != null) { return (T) QueryStatistics.fromPb(statisticPb); } else if (jobConfigPb.getCopy() != null) { return (T) CopyStatistics.fromPb(statisticPb); } else { throw new IllegalArgumentException("unknown job configuration: " + jobConfigPb); } } }
@SuppressWarnings("unchecked") static <T extends JobStatistics> T fromPb( com.google.api.services.bigquery.model.JobStatistics statisticPb) { if (statisticPb.getLoad() != null) { return (T) LoadStatistics.fromPb(statisticPb); } else if (statisticPb.getExtract() != null) { return (T) ExtractStatistics.fromPb(statisticPb); } else if (statisticPb.getQuery() != null) { return (T) QueryStatistics.fromPb(statisticPb); } else { return (T) CopyStatistics.fromPb(statisticPb); } } }
@Override public final int hashCode() { return Objects.hash(baseHashCode(), inputBytes, inputFiles, outputBytes, outputRows); }
@Override public final boolean equals(Object obj) { return obj == this || obj != null && obj.getClass().equals(LoadStatistics.class) && baseEquals((LoadStatistics) obj); }
@Override LoadStatistics build() { return new LoadStatistics(this); } }
@Override public final int hashCode() { return Objects.hash( baseHashCode(), inputBytes, inputFiles, outputBytes, outputRows, badRecords); }
@Override public final boolean equals(Object obj) { return obj == this || obj != null && obj.getClass().equals(LoadStatistics.class) && baseEquals((LoadStatistics) obj); }
@Override LoadStatistics build() { return new LoadStatistics(this); } }
attributes.put(BigQueryAttributes.JOB_NB_RECORDS_ATTR, Long.toString(stats.getOutputRows()));
assertEquals(1L, statistics.getInputFiles().longValue()); assertEquals(2L, statistics.getOutputRows().longValue()); LoadJobConfiguration jobConfiguration = job.getConfiguration(); assertEquals(TABLE_SCHEMA, jobConfiguration.getSchema());
assertEquals(FILE_COUNT, EXTRACT_STATISTICS.getDestinationUriFileCounts()); assertEquals(CREATION_TIME, LOAD_STATISTICS.getCreationTime()); assertEquals(START_TIME, LOAD_STATISTICS.getStartTime()); assertEquals(END_TIME, LOAD_STATISTICS.getEndTime()); assertEquals(INPUT_BYTES, LOAD_STATISTICS.getInputBytes()); assertEquals(INPUT_FILES, LOAD_STATISTICS.getInputFiles()); assertEquals(OUTPUT_BYTES, LOAD_STATISTICS.getOutputBytes()); assertEquals(OUTPUT_ROWS, LOAD_STATISTICS.getOutputRows()); assertEquals(BAD_RECORDS, LOAD_STATISTICS.getBadRecords()); assertEquals(TIMELINE, QUERY_STATISTICS.getTimeline()); assertEquals(CREATION_TIME, LOAD_STATISTICS_INCOMPLETE.getCreationTime()); assertEquals(START_TIME, LOAD_STATISTICS_INCOMPLETE.getStartTime()); assertEquals(END_TIME, LOAD_STATISTICS_INCOMPLETE.getEndTime()); assertEquals(INPUT_BYTES, LOAD_STATISTICS_INCOMPLETE.getInputBytes()); assertEquals(INPUT_FILES, LOAD_STATISTICS_INCOMPLETE.getInputFiles()); assertEquals(BAD_RECORDS, LOAD_STATISTICS_INCOMPLETE.getBadRecords()); assertEquals(null, LOAD_STATISTICS_INCOMPLETE.getOutputBytes()); assertEquals(null, LOAD_STATISTICS_INCOMPLETE.getOutputRows());
/** Example of writing a local file to a table. */ // [TARGET writer(WriteChannelConfiguration)] // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE FileSystems.getDefault().getPath(".", "my-data.csv")] // [VARIABLE "us"] public long writeFileToTable(String datasetName, String tableName, Path csvPath, String location) throws IOException, InterruptedException, TimeoutException { // [START bigquery_load_from_file] TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build(); // The location must be specified; other fields can be auto-detected. JobId jobId = JobId.newBuilder().setLocation(location).build(); TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); // Write data to writer try (OutputStream stream = Channels.newOutputStream(writer)) { Files.copy(csvPath, stream); } // Get load job Job job = writer.getJob(); job = job.waitFor(); LoadStatistics stats = job.getStatistics(); return stats.getOutputRows(); // [END bigquery_load_from_file] }
/** Example of creating a channel with which to write to a table. */ // [TARGET writer(WriteChannelConfiguration)] // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE "StringValue1\nStringValue2\n"] public long writeToTable(String datasetName, String tableName, String csvData) throws IOException, InterruptedException, TimeoutException { // [START ] TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build(); TableDataWriteChannel writer = bigquery.writer(writeChannelConfiguration); // Write data to writer try { writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); } finally { writer.close(); } // Get load job Job job = writer.getJob(); job = job.waitFor(); LoadStatistics stats = job.getStatistics(); return stats.getOutputRows(); // [END ] }
@SuppressWarnings("unchecked") static <T extends JobStatistics> T fromPb(com.google.api.services.bigquery.model.Job jobPb) { JobConfiguration jobConfigPb = jobPb.getConfiguration(); com.google.api.services.bigquery.model.JobStatistics statisticPb = jobPb.getStatistics(); if (jobConfigPb.getLoad() != null) { return (T) LoadStatistics.fromPb(statisticPb); } else if (jobConfigPb.getExtract() != null) { return (T) ExtractStatistics.fromPb(statisticPb); } else if (jobConfigPb.getQuery() != null) { return (T) QueryStatistics.fromPb(statisticPb); } else if (jobConfigPb.getCopy() != null) { return (T) CopyStatistics.fromPb(statisticPb); } else { throw new IllegalArgumentException("unknown job configuration: " + jobConfigPb); } } }
private void compareLoadStatistics(LoadStatistics expected, LoadStatistics value) { assertEquals(expected, value); compareStatistics(expected, value); assertEquals(expected.getInputBytes(), value.getInputBytes()); assertEquals(expected.getInputFiles(), value.getInputFiles()); assertEquals(expected.getOutputBytes(), value.getOutputBytes()); assertEquals(expected.getOutputRows(), value.getOutputRows()); }
@Override LoadStatistics build() { return new LoadStatistics(this); } }
@Test public void testToPbAndFromPb() { compareExtractStatistics( EXTRACT_STATISTICS, ExtractStatistics.fromPb(EXTRACT_STATISTICS.toPb())); compareLoadStatistics(LOAD_STATISTICS, LoadStatistics.fromPb(LOAD_STATISTICS.toPb())); compareQueryStatistics(QUERY_STATISTICS, QueryStatistics.fromPb(QUERY_STATISTICS.toPb())); compareStatistics(COPY_STATISTICS, CopyStatistics.fromPb(COPY_STATISTICS.toPb())); compareLoadStatistics( LOAD_STATISTICS_INCOMPLETE, LoadStatistics.fromPb(LOAD_STATISTICS_INCOMPLETE.toPb())); compareQueryStatistics( QUERY_STATISTICS_INCOMPLETE, QueryStatistics.fromPb(QUERY_STATISTICS_INCOMPLETE.toPb())); }