@Override public Tuple<String, Iterable<com.google.api.services.bigquery.model.Job>> call() { return serviceOptions .getBigQueryRpcV2() .listJobs(serviceOptions.getProjectId(), optionsMap); } },
@Override public Page<Dataset> listDatasets(DatasetListOption... options) { return listDatasets(getOptions().getProjectId(), options); }
@Override public TableDataWriteChannel writer( JobId jobId, WriteChannelConfiguration writeChannelConfiguration) { return new TableDataWriteChannel( getOptions(), jobId.setProjectId(getOptions().getProjectId()), writeChannelConfiguration.setProjectId(getOptions().getProjectId())); }
@Override public Page<Table> listTables(String datasetId, TableListOption... options) { return listTables( DatasetId.of(getOptions().getProjectId(), datasetId), getOptions(), optionMap(options)); }
@Override public boolean delete(TableId tableId) { final TableId completeTableId = tableId.setProjectId( Strings.isNullOrEmpty(tableId.getProject()) ? getOptions().getProjectId() : tableId.getProject()); try { return runWithRetries( new Callable<Boolean>() { @Override public Boolean call() { return bigQueryRpc.deleteTable( completeTableId.getProject(), completeTableId.getDataset(), completeTableId.getTable()); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Override public Job create(Job job, Map<Option, ?> options) { try { String projectId = job.getJobReference() != null ? job.getJobReference().getProjectId() : this.options.getProjectId(); return bigquery .jobs() .insert(projectId, job) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
@Override public boolean delete(DatasetId datasetId, DatasetDeleteOption... options) { final DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); try { return runWithRetries( new Callable<Boolean>() { @Override public Boolean call() { return bigQueryRpc.deleteDataset( completeDatasetId.getProject(), completeDatasetId.getDataset(), optionsMap); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Override public Dataset getDataset(final DatasetId datasetId, DatasetOption... options) { final DatasetId completeDatasetId = datasetId.setProjectId(getOptions().getProjectId()); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); try { com.google.api.services.bigquery.model.Dataset answer = runWithRetries( new Callable<com.google.api.services.bigquery.model.Dataset>() { @Override public com.google.api.services.bigquery.model.Dataset call() { return bigQueryRpc.getDataset( completeDatasetId.getProject(), completeDatasetId.getDataset(), optionsMap); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); return answer == null ? null : Dataset.fromPb(this, answer); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Override public Dataset create(DatasetInfo datasetInfo, DatasetOption... options) { final com.google.api.services.bigquery.model.Dataset datasetPb = datasetInfo.setProjectId(getOptions().getProjectId()).toPb(); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); try { return Dataset.fromPb( this, runWithRetries( new Callable<com.google.api.services.bigquery.model.Dataset>() { @Override public com.google.api.services.bigquery.model.Dataset call() { return bigQueryRpc.create(datasetPb, optionsMap); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock())); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Override public Dataset update(DatasetInfo datasetInfo, DatasetOption... options) { final com.google.api.services.bigquery.model.Dataset datasetPb = datasetInfo.setProjectId(getOptions().getProjectId()).toPb(); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); try { return Dataset.fromPb( this, runWithRetries( new Callable<com.google.api.services.bigquery.model.Dataset>() { @Override public com.google.api.services.bigquery.model.Dataset call() { return bigQueryRpc.patch(datasetPb, optionsMap); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock())); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Override public Table update(TableInfo tableInfo, TableOption... options) { final com.google.api.services.bigquery.model.Table tablePb = tableInfo .setProjectId( Strings.isNullOrEmpty(tableInfo.getTableId().getProject()) ? getOptions().getProjectId() : tableInfo.getTableId().getProject()) .toPb(); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); try { return Table.fromPb( this, runWithRetries( new Callable<com.google.api.services.bigquery.model.Table>() { @Override public com.google.api.services.bigquery.model.Table call() { return bigQueryRpc.patch(tablePb, optionsMap); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock())); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Override public String open(Job loadJob) { try { String builder = BASE_RESUMABLE_URI + options.getProjectId() + "/jobs"; GenericUrl url = new GenericUrl(builder); url.set("uploadType", "resumable"); JsonFactory jsonFactory = bigquery.getJsonFactory(); HttpRequestFactory requestFactory = bigquery.getRequestFactory(); HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, loadJob)); httpRequest.getHeaders().set("X-Upload-Content-Value", "application/octet-stream"); HttpResponse response = httpRequest.execute(); return response.getHeaders().getLocation(); } catch (IOException ex) { throw translate(ex); } }
@Override public Table create(TableInfo tableInfo, TableOption... options) { final com.google.api.services.bigquery.model.Table tablePb = tableInfo .setProjectId( Strings.isNullOrEmpty(tableInfo.getTableId().getProject()) ? getOptions().getProjectId() : tableInfo.getTableId().getProject()) .toPb(); final Map<BigQueryRpc.Option, ?> optionsMap = optionMap(options); try { return Table.fromPb( this, runWithRetries( new Callable<com.google.api.services.bigquery.model.Table>() { @Override public com.google.api.services.bigquery.model.Table call() { return bigQueryRpc.create(tablePb, optionsMap); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock())); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
@Test public void testUpdateDataset() { Dataset dataset = bigquery.create( DatasetInfo.newBuilder(OTHER_DATASET) .setDescription("Some Description") .setLabels(Collections.singletonMap("a", "b")) .build()); assertThat(dataset).isNotNull(); assertThat(dataset.getDatasetId().getProject()).isEqualTo(bigquery.getOptions().getProjectId()); assertThat(dataset.getDatasetId().getDataset()).isEqualTo(OTHER_DATASET); assertThat(dataset.getDescription()).isEqualTo("Some Description"); assertThat(dataset.getLabels()).containsExactly("a", "b"); Map<String, String> updateLabels = new HashMap<>(); updateLabels.put("x", "y"); updateLabels.put("a", null); Dataset updatedDataset = bigquery.update( dataset .toBuilder() .setDescription("Updated Description") .setLabels(updateLabels) .build()); assertThat(updatedDataset.getDescription()).isEqualTo("Updated Description"); assertThat(updatedDataset.getLabels()).containsExactly("x", "y"); updatedDataset = bigquery.update(updatedDataset.toBuilder().setLabels(null).build()); assertThat(updatedDataset.getLabels()).isEmpty(); assertThat(dataset.delete()).isTrue(); }
@Override public boolean cancel(JobId jobId) { final JobId completeJobId = jobId .setProjectId(getOptions().getProjectId()) .setLocation( jobId.getLocation() == null && getOptions().getLocation() != null ? getOptions().getLocation() : jobId.getLocation()); try { return runWithRetries( new Callable<Boolean>() { @Override public Boolean call() { return bigQueryRpc.cancel( completeJobId.getProject(), completeJobId.getJob(), completeJobId.getLocation()); } }, getOptions().getRetrySettings(), EXCEPTION_HANDLER, getOptions().getClock()); } catch (RetryHelper.RetryHelperException e) { throw BigQueryException.translateAndThrow(e); } }
final JobId completeJobId = jobId .setProjectId(getOptions().getProjectId()) .setLocation( jobId.getLocation() == null && getOptions().getLocation() != null
@Test public void testGetDataset() { Dataset dataset = bigquery.getDataset(DATASET); assertEquals(bigquery.getOptions().getProjectId(), dataset.getDatasetId().getProject()); assertEquals(DATASET, dataset.getDatasetId().getDataset()); assertEquals(DESCRIPTION, dataset.getDescription()); assertEquals(LABELS, dataset.getLabels()); assertNotNull(dataset.getAcl()); assertNotNull(dataset.getEtag()); assertNotNull(dataset.getGeneratedId()); assertNotNull(dataset.getLastModified()); assertNotNull(dataset.getSelfLink()); }
@Test public void testCreateFromStream() { RemoteBigQueryHelper helper = RemoteBigQueryHelper.create(PROJECT_ID, JSON_KEY_STREAM); BigQueryOptions options = helper.getOptions(); assertEquals(PROJECT_ID, options.getProjectId()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getConnectTimeout()); assertEquals(60000, ((HttpTransportOptions) options.getTransportOptions()).getReadTimeout()); assertEquals(10, options.getRetrySettings().getMaxAttempts()); assertEquals(Duration.ofMillis(30000), options.getRetrySettings().getMaxRetryDelay()); assertEquals(Duration.ofMillis(120000), options.getRetrySettings().getTotalTimeout()); assertEquals(Duration.ofMillis(250), options.getRetrySettings().getInitialRetryDelay()); } }
@Test public void testGetDatasetWithSelectedFields() { Dataset dataset = bigquery.getDataset( DATASET, DatasetOption.fields(DatasetField.CREATION_TIME, DatasetField.LABELS)); assertEquals(bigquery.getOptions().getProjectId(), dataset.getDatasetId().getProject()); assertEquals(DATASET, dataset.getDatasetId().getDataset()); assertEquals(LABELS, dataset.getLabels()); assertNotNull(dataset.getCreationTime()); assertNull(dataset.getDescription()); assertNull(dataset.getDefaultTableLifetime()); assertNull(dataset.getAcl()); assertNull(dataset.getEtag()); assertNull(dataset.getFriendlyName()); assertNull(dataset.getGeneratedId()); assertNull(dataset.getLastModified()); assertNull(dataset.getLocation()); assertNull(dataset.getSelfLink()); }
@Test public void testUpdateDatasetWithSelectedFields() { Dataset dataset = bigquery.create( DatasetInfo.newBuilder(OTHER_DATASET).setDescription("Some Description").build()); assertNotNull(dataset); assertEquals(bigquery.getOptions().getProjectId(), dataset.getDatasetId().getProject()); assertEquals(OTHER_DATASET, dataset.getDatasetId().getDataset()); assertEquals("Some Description", dataset.getDescription()); Dataset updatedDataset = bigquery.update( dataset.toBuilder().setDescription("Updated Description").build(), DatasetOption.fields(DatasetField.DESCRIPTION)); assertEquals("Updated Description", updatedDataset.getDescription()); assertNull(updatedDataset.getCreationTime()); assertNull(updatedDataset.getDefaultTableLifetime()); assertNull(updatedDataset.getAcl()); assertNull(updatedDataset.getEtag()); assertNull(updatedDataset.getFriendlyName()); assertNull(updatedDataset.getGeneratedId()); assertNull(updatedDataset.getLastModified()); assertNull(updatedDataset.getLocation()); assertNull(updatedDataset.getSelfLink()); assertTrue(dataset.delete()); }