Table toPb() { Table tablePb = new Table(); if (getSchema() != null) { tablePb.setSchema(getSchema().toPb()); } tablePb.setType(getType().name()); return tablePb; }
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, JsonNode node) { if (node.isTextual()) { return new Table() .setTableReference(Bq.tableReference(defaultProjectId, defaultDataset, node.asText())); } else { TableConfig config; try { config = objectMapper.readValue(node.traverse(), TableConfig.class); } catch (IOException e) { throw new ConfigException("Invalid table reference or configuration: " + node, e); } return table(defaultProjectId, defaultDataset, config); } }
public Table toTable(BigQueryTable source) { Table sink = new Table(); TableReference tableRef = tableReference(source); TableSchema schema = toTableSchema(source); sink.setTableReference(tableRef); sink.setSchema(schema); return sink; }
Table toPb() { Table tablePb = new Table(); if (schema != null) { tablePb.setSchema(schema.toPb()); } tablePb.setType(type.name()); return tablePb; }
Table toPb() { Table tablePb = new Table(); if (schema != null) { tablePb.setSchema(schema.toPb()); } tablePb.setType(type.name()); return tablePb; }
@Override public Table patchTableDescription( TableReference tableReference, @Nullable String tableDescription) throws IOException, InterruptedException { Table table = new Table(); table.setDescription(tableDescription); return executeWithRetries( client .tables() .patch( tableReference.getProjectId(), tableReference.getDatasetId(), tableReference.getTableId(), table), String.format( "Unable to patch table description: %s, aborting after %d retries.", tableReference, MAX_RPC_RETRIES), Sleeper.DEFAULT, createDefaultBackoff(), ALWAYS_RETRY); } }
private Table table(String defaultProjectId, Optional<DatasetReference> defaultDataset, TableConfig config) { Optional<String> datasetId = config.dataset().or(defaultDataset.transform(DatasetReference::getDatasetId)); if (!datasetId.isPresent()) { throw new ConfigException("Bad table reference or configuration: Missing 'dataset'"); } return new Table() .setTableReference(new TableReference() .setProjectId(config.project().or(defaultProjectId)) .setDatasetId(datasetId.get()) .setTableId(config.id())) .setSchema(config.schema().orNull()) .setFriendlyName(config.friendly_name().orNull()) .setExpirationTime(config.expiration_time() .transform(p -> p.getTimestamp().toInstant(request.getTimeZone()).toEpochMilli()).orNull()) .setTimePartitioning(config.time_partitioning().orNull()) .setView(config.view().orNull()); }
Table toPb() { Table tablePb = new Table(); if (getSchema() != null) { tablePb.setSchema(getSchema().toPb()); } tablePb.setType(getType().name()); return tablePb; }
@Test public void testExecuteWithRetries() throws IOException, InterruptedException { Table testTable = new Table(); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testTable)); Table table = BigQueryServicesImpl.executeWithRetries( bigquery.tables().get("projectId", "datasetId", "tableId"), "Failed to get table.", Sleeper.DEFAULT, BackOff.STOP_BACKOFF, BigQueryServicesImpl.ALWAYS_RETRY); assertEquals(testTable, table); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
@Test public void testCreateTableSucceeds() throws IOException { TableReference ref = new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table"); Table testTable = new Table().setTableReference(ref); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(200); when(response.getContent()).thenReturn(toStream(testTable)); BigQueryServicesImpl.DatasetServiceImpl services = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); Table ret = services.tryCreateTable( testTable, new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF), Sleeper.DEFAULT); assertEquals(testTable, ret); verify(response, times(1)).getStatusCode(); verify(response, times(1)).getContent(); verify(response, times(1)).getContentType(); }
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
@Test public void testGetTableSucceeds() throws Exception { TableReference tableRef = new TableReference() .setProjectId("projectId") .setDatasetId("datasetId") .setTableId("tableId"); Table testTable = new Table(); testTable.setTableReference(tableRef); when(response.getContentType()).thenReturn(Json.MEDIA_TYPE); when(response.getStatusCode()).thenReturn(403).thenReturn(200); when(response.getContent()) .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403))) .thenReturn(toStream(testTable)); BigQueryServicesImpl.DatasetServiceImpl datasetService = new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create()); Table table = datasetService.getTable(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT); assertEquals(testTable, table); verify(response, times(2)).getStatusCode(); verify(response, times(2)).getContent(); verify(response, times(2)).getContentType(); }
@Override public Table apply(TableList.Tables tablePb) { return new Table() .setFriendlyName(tablePb.getFriendlyName()) .setId(tablePb.getId()) .setKind(tablePb.getKind()) .setTableReference(tablePb.getTableReference()) .setType(tablePb.getType()); } }));
public static Table createTable(String projectId, String datasetId, String tableId, TableSchema schema, Bigquery bigquery) throws IOException { Table table = new Table(); TableReference tableRef = new TableReference(); tableRef.setDatasetId(datasetId); tableRef.setProjectId(projectId); tableRef.setTableId(tableId); table.setTableReference(tableRef); table.setFriendlyName(tableId); table.setSchema(schema); try { return bigquery.tables().insert(projectId, datasetId, table).execute(); } catch (GoogleJsonResponseException e) { return null; //table already exists } } }
private Table basicTableSchema() { return new Table() .setSchema( new TableSchema() .setFields( Arrays.asList( new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("answer").setType("INTEGER")))); }
private Table toTable(Shape shape, GoogleBigQueryTable dataSource) { Table table = new Table(); BigQueryTableReference ref = dataSource.getTableReference(); TableReference reference = new TableReference(); reference.setProjectId(ref.getProjectId()); reference.setDatasetId(ref.getDatasetId()); reference.setTableId(ref.getTableId()); table.setTableReference(reference); if (!(dataSource instanceof GoogleBigQueryView)){ TableSchema tableSchema = tableGenerator.toTableSchema(shape); table.setSchema(tableSchema); } ExternalDataConfiguration external = dataSource.getExternalDataConfiguration(); if (external != null) { table.setExternalDataConfiguration(external); table.setType("EXTERNAL"); } return table; }
private JobStatus runQueryJob(JobConfigurationQuery query) throws IOException, InterruptedException { List<TableRow> rows = FakeBigQueryServices.rowsFromEncodedQuery(query.getQuery()); datasetService.createTable(new Table().setTableReference(query.getDestinationTable())); datasetService.insertAll(query.getDestinationTable(), rows, null); return new JobStatus().setState("DONE"); }
private void setupNewTypesQueryTest() { this.bqClient.createNewTable( this.project, this.bigQueryDatasetId, new Table() .setSchema(BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_SCHEMA) .setTableReference( new TableReference() .setTableId(BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_NAME) .setDatasetId(this.bigQueryDatasetId) .setProjectId(this.project))); this.bqClient.insertDataToTable( this.project, this.bigQueryDatasetId, BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_NAME, BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_DATA); this.options.setQuery( String.format( "SELECT bytes, date, time FROM [%s:%s.%s]", project, this.bigQueryDatasetId, BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_NAME)); this.options.setOutput(outputTable); this.options.setOutputSchema(BigQueryToTableIT.NEW_TYPES_QUERY_TABLE_SCHEMA); }