/** Example of creating a channel with which to write to a table. */ // [TARGET writer(WriteChannelConfiguration)] // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE "StringValue1\nStringValue2\n"] public long writeToTable(String datasetName, String tableName, String csvData) throws IOException, InterruptedException, TimeoutException { // [START ] TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build(); TableDataWriteChannel writer = bigquery.writer(writeChannelConfiguration); // Write data to writer try { writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); } finally { writer.close(); } // Get load job Job job = writer.getJob(); job = job.waitFor(); LoadStatistics stats = job.getStatistics(); return stats.getOutputRows(); // [END ] }
/** Example of creating a channel with which to write to a table. */ // [TARGET writer(JobId, WriteChannelConfiguration)] // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE "StringValue1\nStringValue2\n"] // [VARIABLE "us"] public long writeToTableLocation( String datasetName, String tableName, String csvData, String location) throws IOException, InterruptedException, TimeoutException { // [START ] TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build(); // The location must be specified; other fields can be auto-detected. JobId jobId = JobId.newBuilder().setLocation(location).build(); TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); // Write data to writer try { writer.write(ByteBuffer.wrap(csvData.getBytes(Charsets.UTF_8))); } finally { writer.close(); } // Get load job Job job = writer.getJob(); job = job.waitFor(); LoadStatistics stats = job.getStatistics(); return stats.getOutputRows(); // [END ] }
/** Example of writing a local file to a table. */ // [TARGET writer(WriteChannelConfiguration)] // [VARIABLE "my_dataset_name"] // [VARIABLE "my_table_name"] // [VARIABLE FileSystems.getDefault().getPath(".", "my-data.csv")] // [VARIABLE "us"] public long writeFileToTable(String datasetName, String tableName, Path csvPath, String location) throws IOException, InterruptedException, TimeoutException { // [START bigquery_load_from_file] TableId tableId = TableId.of(datasetName, tableName); WriteChannelConfiguration writeChannelConfiguration = WriteChannelConfiguration.newBuilder(tableId).setFormatOptions(FormatOptions.csv()).build(); // The location must be specified; other fields can be auto-detected. JobId jobId = JobId.newBuilder().setLocation(location).build(); TableDataWriteChannel writer = bigquery.writer(jobId, writeChannelConfiguration); // Write data to writer try (OutputStream stream = Channels.newOutputStream(writer)) { Files.copy(csvPath, stream); } // Get load job Job job = writer.getJob(); job = job.waitFor(); LoadStatistics stats = job.getStatistics(); return stats.getOutputRows(); // [END bigquery_load_from_file] }
final Map<String, String> attributes = new HashMap<>(); attributes.put(BigQueryAttributes.JOB_CREATE_TIME_ATTR, Long.toString(job.getStatistics().getCreationTime())); attributes.put(BigQueryAttributes.JOB_END_TIME_ATTR, Long.toString(job.getStatistics().getEndTime())); attributes.put(BigQueryAttributes.JOB_START_TIME_ATTR, Long.toString(job.getStatistics().getStartTime())); attributes.put(BigQueryAttributes.JOB_LINK_ATTR, job.getSelfLink()); if(job.getStatistics() instanceof LoadStatistics) { final LoadStatistics stats = (LoadStatistics) job.getStatistics(); attributes.put(BigQueryAttributes.JOB_NB_RECORDS_ATTR, Long.toString(stats.getOutputRows())); session.transfer(flowFile, REL_FAILURE); } else { session.getProvenanceReporter().send(flowFile, job.getSelfLink(), job.getStatistics().getEndTime() - job.getStatistics().getStartTime()); session.transfer(flowFile, REL_SUCCESS);
@Test public void testListJobs() { Page<Job> jobs = bigquery.listJobs(); for (Job job : jobs.getValues()) { assertNotNull(job.getJobId()); assertNotNull(job.getStatistics()); assertNotNull(job.getStatus()); assertNotNull(job.getUserEmail()); assertNotNull(job.getGeneratedId()); } }
@Test public void testListJobsWithSelectedFields() { Page<Job> jobs = bigquery.listJobs(JobListOption.fields(JobField.USER_EMAIL)); for (Job job : jobs.getValues()) { assertNotNull(job.getJobId()); assertNotNull(job.getStatus()); assertNotNull(job.getUserEmail()); assertNull(job.getStatistics()); assertNull(job.getGeneratedId()); } }
@Test public void testBuilder() { initializeExpectedJob(2); replay(bigquery); Job builtJob = new Job.Builder(serviceMockReturnsOptions, COPY_CONFIGURATION) .setJobId(JOB_ID) .setStatistics(COPY_JOB_STATISTICS) .setJobId(JOB_ID) .setEtag(ETAG) .setGeneratedId(GENERATED_ID) .setSelfLink(SELF_LINK) .setUserEmail(EMAIL) .setStatus(JOB_STATUS) .build(); assertEquals(ETAG, builtJob.getEtag()); assertEquals(GENERATED_ID, builtJob.getGeneratedId()); assertEquals(SELF_LINK, builtJob.getSelfLink()); assertEquals(EMAIL, builtJob.getUserEmail()); assertEquals(JOB_ID, builtJob.getJobId()); assertEquals(JOB_STATUS, builtJob.getStatus()); assertEquals(COPY_CONFIGURATION, builtJob.getConfiguration()); assertEquals(COPY_JOB_STATISTICS, builtJob.getStatistics()); assertSame(serviceMockReturnsOptions, builtJob.getBigQuery()); }
LoadStatistics statistics = job.getStatistics(); assertEquals(1L, statistics.getInputFiles().longValue()); assertEquals(2L, statistics.getOutputRows().longValue());
@Test public void testQuery() throws InterruptedException { String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.getTable(); QueryJobConfiguration config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(DatasetId.of(DATASET)).build(); Job job = bigquery.create(JobInfo.of(JobId.of(), config)); TableResult result = job.getQueryResults(); assertEquals(QUERY_RESULT_SCHEMA, result.getSchema()); int rowCount = 0; for (FieldValueList row : result.getValues()) { FieldValue timestampCell = row.get(0); assertEquals(timestampCell, row.get("TimestampField")); FieldValue stringCell = row.get(1); assertEquals(stringCell, row.get("StringField")); FieldValue booleanCell = row.get(2); assertEquals(booleanCell, row.get("BooleanField")); assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.getAttribute()); assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.getAttribute()); assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.getAttribute()); assertEquals(1408452095220000L, timestampCell.getTimestampValue()); assertEquals("stringValue", stringCell.getStringValue()); assertEquals(false, booleanCell.getBooleanValue()); rowCount++; } assertEquals(2, rowCount); Job job2 = bigquery.getJob(job.getJobId()); JobStatistics.QueryStatistics statistics = job2.getStatistics(); assertNotNull(statistics.getQueryPlan()); }
assertTrue(bigquery.delete(DATASET, tableName)); Job queryJob = bigquery.getJob(remoteJob.getJobId()); JobStatistics.QueryStatistics statistics = queryJob.getStatistics(); assertNotNull(statistics.getQueryPlan());
assertNotNull(createdConfiguration.getDestinationTable()); assertNotNull(createdJob.getEtag()); assertNull(createdJob.getStatistics()); assertNull(createdJob.getStatus()); assertNull(createdJob.getSelfLink()); createdConfiguration.getWriteDisposition(), remoteConfiguration.getWriteDisposition()); assertNotNull(remoteJob.getEtag()); assertNull(remoteJob.getStatistics()); assertNull(remoteJob.getStatus()); assertNull(remoteJob.getSelfLink());
createdConfiguration.getWriteDisposition(), remoteConfiguration.getWriteDisposition()); assertNotNull(remoteJob.getEtag()); assertNotNull(remoteJob.getStatistics()); assertNotNull(remoteJob.getStatus()); assertEquals(createdJob.getSelfLink(), remoteJob.getSelfLink());