Dataset toPb() { Dataset datasetPb = new Dataset(); datasetPb.setDatasetReference(datasetId.toPb()); datasetPb.setCreationTime(creationTime); datasetPb.setDefaultTableExpirationMs(defaultTableLifetime); datasetPb.setDescription(description); datasetPb.setEtag(etag); datasetPb.setFriendlyName(friendlyName); datasetPb.setId(generatedId); datasetPb.setLastModifiedTime(lastModified); datasetPb.setLocation(location); datasetPb.setSelfLink(selfLink); if (acl != null) { datasetPb.setAccess( Lists.transform( acl, new Function<Acl, Dataset.Access>() { @Override public Dataset.Access apply(Acl acl) { return acl.toPb(); } })); } datasetPb.setLabels(labels.toPb()); return datasetPb; }
BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) { if (datasetPb.getDatasetReference() != null) { this.datasetId = DatasetId.fromPb(datasetPb.getDatasetReference()); } if (datasetPb.getAccess() != null) { this.acl = Lists.transform( datasetPb.getAccess(), new Function<Dataset.Access, Acl>() { @Override public Acl apply(Dataset.Access accessPb) { return Acl.fromPb(accessPb); } }); } this.creationTime = datasetPb.getCreationTime(); this.defaultTableLifetime = datasetPb.getDefaultTableExpirationMs(); this.description = datasetPb.getDescription(); this.etag = datasetPb.getEtag(); this.friendlyName = datasetPb.getFriendlyName(); this.generatedId = datasetPb.getId(); this.lastModified = datasetPb.getLastModifiedTime(); this.location = datasetPb.getLocation(); this.selfLink = datasetPb.getSelfLink(); this.labels = Labels.fromPb(datasetPb.getLabels()); }
@Override public Dataset apply(DatasetList.Datasets datasetPb) { return new Dataset() .setDatasetReference(datasetPb.getDatasetReference()) .setFriendlyName(datasetPb.getFriendlyName()) .setId(datasetPb.getId()) .setKind(datasetPb.getKind()) .setLabels(datasetPb.getLabels()); } };
private Dataset dataset(String defaultProjectId, DatasetConfig config) { return new Dataset() .setDatasetReference(new DatasetReference() .setProjectId(config.project().or(defaultProjectId)) .setDatasetId(config.id())) .setFriendlyName(config.friendly_name().orNull()) .setDefaultTableExpirationMs(config.default_table_expiration().transform(d -> d.getDuration().toMillis()).orNull()) .setLocation(config.location().orNull()) .setAccess(config.access().orNull()) .setLabels(config.labels().orNull()); }
new DatasetReference().setProjectId(projectId).setDatasetId(datasetId); Dataset dataset = new Dataset().setDatasetReference(datasetRef); if (location != null) { dataset.setLocation(location); dataset.setFriendlyName(description); dataset.setDescription(description); dataset.setDefaultTableExpirationMs(defaultTableExpirationMs);
public void createNewDataset(String projectId, String datasetId) { try { bqClient .datasets() .insert( projectId, new Dataset().setDatasetReference(new DatasetReference().setDatasetId(datasetId))) .execute(); LOG.info("Successfully created new dataset : " + datasetId); } catch (Exception e) { LOG.debug("Exceptions caught when creating new dataset: " + e.getMessage()); } }
private void createDataset(String datasetId, @Nullable String location) throws IOException, InterruptedException { Dataset dataset = new Dataset(); DatasetReference reference = new DatasetReference(); reference.setProjectId(projectId); reference.setDatasetId(datasetId); dataset.setDatasetReference(reference); if (location != null) { dataset.setLocation(location); } executeWithBackOff( client.datasets().insert(projectId, dataset), String.format( "Error when trying to create the temporary dataset %s in project %s.", datasetId, projectId)); }
@Test public void testListToDataset() { DatasetReference datasetRef = new DatasetReference().setDatasetId("dataset-id").setProjectId("project-id"); DatasetList.Datasets listDataSet = new DatasetList.Datasets() .setDatasetReference(datasetRef) .setId("project-id:dataset-id") .setFriendlyName("friendly") .setKind("bigquery#dataset") .setLabels(Collections.singletonMap("foo", "bar")); Dataset dataset = HttpBigQueryRpc.LIST_TO_DATASET.apply(listDataSet); assertThat(dataset.getKind()).isEqualTo("bigquery#dataset"); assertThat(dataset.getId()).isEqualTo("project-id:dataset-id"); assertThat(dataset.getFriendlyName()).isEqualTo("friendly"); assertThat(dataset.getDatasetReference()).isEqualTo(datasetRef); assertThat(dataset.getLabels()).containsExactly("foo", "bar"); } }
@Override public Dataset create(Dataset dataset, Map<Option, ?> options) { try { return bigquery .datasets() .insert(dataset.getDatasetReference().getProjectId(), dataset) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
static String getDatasetLocation( DatasetService datasetService, String projectId, String datasetId) { Dataset dataset; try { dataset = datasetService.getDataset(projectId, datasetId); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } throw new RuntimeException( String.format( "unable to obtain dataset for dataset %s in project %s", datasetId, projectId), e); } return dataset.getLocation(); }
@Override public Dataset apply(DatasetList.Datasets datasetPb) { return new Dataset() .setDatasetReference(datasetPb.getDatasetReference()) .setFriendlyName(datasetPb.getFriendlyName()) .setId(datasetPb.getId()) .setKind(datasetPb.getKind()); } }));
private Dataset dataset(String defaultProjectId, JsonNode node) { if (node.isTextual()) { return new Dataset() .setDatasetReference(datasetReference(defaultProjectId, node.asText())); } else { DatasetConfig config; try { config = objectMapper.readValue(node.traverse(), DatasetConfig.class); } catch (IOException e) { throw new ConfigException("Invalid dataset reference or configuration: " + node, e); } return dataset(defaultProjectId, config); } }
/** * Creates the temporary dataset that will contain all of the task work tables. * * @param context the job's context. * @throws IOException on IO Error. */ @Override public void setupJob(JobContext context) throws IOException { logger.atFine().log("setupJob(%s)", lazy(() -> HadoopToStringUtil.toString(context))); // Create dataset. DatasetReference datasetReference = new DatasetReference(); datasetReference.setProjectId(tempTableRef.getProjectId()); datasetReference.setDatasetId(tempTableRef.getDatasetId()); Dataset tempDataset = new Dataset(); tempDataset.setDatasetReference(datasetReference); tempDataset.setLocation(getLocation(context)); // Insert dataset into Bigquery. Bigquery.Datasets datasets = bigQueryHelper.getRawBigquery().datasets(); // TODO(user): Maybe allow the dataset to exist already instead of throwing 409 here. logger.atFine().log( "Creating temporary dataset '%s' for project '%s'", tempTableRef.getDatasetId(), tempTableRef.getProjectId()); // NB: Even though this "insert" makes it look like we can specify a different projectId than // the one which owns the dataset, it actually has to match. datasets.insert(tempTableRef.getProjectId(), tempDataset).execute(); }
@Override public Dataset patch(Dataset dataset, Map<Option, ?> options) { try { DatasetReference reference = dataset.getDatasetReference(); return bigquery .datasets() .patch(reference.getProjectId(), reference.getDatasetId(), dataset) .setFields(Option.FIELDS.getString(options)) .execute(); } catch (IOException ex) { throw translate(ex); } }
createJobReference(projectId, "direct-bigqueryhelper-import", dataset.getLocation()); Job job = new Job(); job.setConfiguration(config);
Dataset toPb() { Dataset datasetPb = new Dataset(); datasetPb.setDatasetReference(datasetId.toPb()); datasetPb.setCreationTime(creationTime); datasetPb.setDefaultTableExpirationMs(defaultTableLifetime); datasetPb.setDescription(description); datasetPb.setEtag(etag); datasetPb.setFriendlyName(friendlyName); datasetPb.setId(generatedId); datasetPb.setLastModifiedTime(lastModified); datasetPb.setLocation(location); datasetPb.setSelfLink(selfLink); if (acl != null) { datasetPb.setAccess(Lists.transform(acl, new Function<Acl, Dataset.Access>() { @Override public Dataset.Access apply(Acl acl) { return acl.toPb(); } })); } return datasetPb; }
BuilderImpl(com.google.api.services.bigquery.model.Dataset datasetPb) { if (datasetPb.getDatasetReference() != null) { this.datasetId = DatasetId.fromPb(datasetPb.getDatasetReference()); } if (datasetPb.getAccess() != null) { this.acl = Lists.transform(datasetPb.getAccess(), new Function<Dataset.Access, Acl>() { @Override public Acl apply(Dataset.Access accessPb) { return Acl.fromPb(accessPb); } }); } this.creationTime = datasetPb.getCreationTime(); this.defaultTableLifetime = datasetPb.getDefaultTableExpirationMs(); this.description = datasetPb.getDescription(); this.etag = datasetPb.getEtag(); this.friendlyName = datasetPb.getFriendlyName(); this.generatedId = datasetPb.getId(); this.lastModified = datasetPb.getLastModifiedTime(); this.location = datasetPb.getLocation(); this.selfLink = datasetPb.getSelfLink(); }
@Override public Dataset apply(DatasetList.Datasets datasetPb) { return new Dataset() .setDatasetReference(datasetPb.getDatasetReference()) .setFriendlyName(datasetPb.getFriendlyName()) .setId(datasetPb.getId()) .setKind(datasetPb.getKind()); } }));
@Override public Dataset getDataset(String projectId, String datasetId) throws IOException, InterruptedException { synchronized (tables) { Map<String, TableContainer> dataset = tables.get(projectId, datasetId); if (dataset == null) { throwNotFound( "Tried to get a dataset %s:%s, but no such table was set", projectId, datasetId); } return new Dataset() .setDatasetReference( new DatasetReference().setDatasetId(datasetId).setProjectId(projectId)); } }
void emptyDataset(String projectId, Dataset dataset) throws IOException { String datasetId = dataset.getDatasetReference().getDatasetId(); deleteDataset(projectId, datasetId); createDataset(projectId, dataset); }