@Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed); queryStatisticsPb.setEstimatedBytesProcessed(estimatedBytesProcessed); queryStatisticsPb.setTotalBytesBilled(totalBytesBilled); queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed); queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed); queryStatisticsPb.setTotalSlotMs(totalSlotMs); if (ddlTargetTable != null) { queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb()); queryStatisticsPb.setReferencedTables( Lists.transform(referencedTables, TableId.TO_PB_FUNCTION)); queryStatisticsPb.setStatementType(statementType.toString()); queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); queryStatisticsPb.setTimeline(Lists.transform(timeline, TimelineSample.TO_PB_FUNCTION)); queryStatisticsPb.setSchema(schema.toPb());
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getQuery() != null) { this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed(); if (statisticsPb.getQuery().getDdlTargetTable() != null) { this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable()); this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed(); this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows(); this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled(); this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed(); this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed(); this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs(); if (statisticsPb.getQuery().getStatementType() != null) { this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType()); if (statisticsPb.getQuery().getReferencedTables() != null) { this.referencedTables = Lists.transform( statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION); if (statisticsPb.getQuery().getQueryPlan() != null) { this.queryPlan = Lists.transform( statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION); if (statisticsPb.getQuery().getTimeline() != null) { this.timeline =
@Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setTotalBytesBilled(totalBytesBilled); queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed); if (queryPlan != null) { queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); } return super.toPb().setQuery(queryStatisticsPb); }
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled(); this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed(); if (statisticsPb.getQuery().getQueryPlan() != null) { this.queryPlan = Lists.transform(statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION); } }
JobStatistics2 queryStats = new JobStatistics2(); queryStats.setReferencedTables(ImmutableList.of(dryRunTable)); queryJobStats.setQuery(queryStats); queryJob.setStatus(new JobStatus()).setStatistics(queryJobStats); new JobStatistics() .setQuery( new JobStatistics2() .setTotalBytesProcessed(100L) .setReferencedTables(ImmutableList.of(table.getTableReference()))));
@Test public void testBigQueryQuerySourceEstimatedSize() throws Exception { List<TableRow> data = ImmutableList.of( new TableRow().set("name", "A").set("number", 10L), new TableRow().set("name", "B").set("number", 11L), new TableRow().set("name", "C").set("number", 12L)); PipelineOptions options = PipelineOptionsFactory.create(); BigQueryOptions bqOptions = options.as(BigQueryOptions.class); bqOptions.setProject("project"); String stepUuid = "testStepUuid"; String query = FakeBigQueryServices.encodeQuery(data); BigQueryQuerySource<TableRow> bqSource = BigQueryQuerySource.create( stepUuid, ValueProvider.StaticValueProvider.of(query), true /* flattenResults */, true /* useLegacySql */, fakeBqServices, TableRowJsonCoder.of(), BigQueryIO.TableRowParser.INSTANCE, QueryPriority.BATCH, null); fakeJobService.expectDryRunQuery( bqOptions.getProject(), query, new JobStatistics().setQuery(new JobStatistics2().setTotalBytesProcessed(100L))); assertEquals(100, bqSource.getEstimatedSizeBytes(bqOptions)); }
@Override public long getEstimatedSizeBytes(PipelineOptions options) throws Exception { BigQueryOptions bqOptions = options.as(BigQueryOptions.class); return dryRunQueryIfNeeded(bqOptions).getQuery().getTotalBytesProcessed(); }
@Nullable List<TableReference> tables = jobStats.getQuery().getReferencedTables(); if (tables != null && !tables.isEmpty()) { Table table = getTable(tables.get(0));
@Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setTotalBytesBilled(totalBytesBilled); queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed); if (queryPlan != null) { queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); } return super.toPb().setQuery(queryStatisticsPb); }
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled(); this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed(); if (statisticsPb.getQuery().getQueryPlan() != null) { this.queryPlan = Lists.transform(statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION); } }
JobStatistics2 queryStats = new JobStatistics2(); queryStats.setReferencedTables(ImmutableList.of(dryRunTable)); queryJobStats.setQuery(queryStats); queryJob.setStatus(new JobStatus()).setStatistics(queryJobStats); new JobStatistics() .setQuery( new JobStatistics2() .setTotalBytesProcessed(100L) .setReferencedTables(ImmutableList.of(queryTable))));
dryRunQueryIfNeeded(bqOptions).getQuery().getReferencedTables(); if (referencedTables != null && !referencedTables.isEmpty()) { TableReference queryTable = referencedTables.get(0);
@Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setDdlOperationPerformed(ddlOperationPerformed); queryStatisticsPb.setEstimatedBytesProcessed(estimatedBytesProcessed); queryStatisticsPb.setTotalBytesBilled(totalBytesBilled); queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed); queryStatisticsPb.setTotalPartitionsProcessed(totalPartitionsProcessed); queryStatisticsPb.setTotalSlotMs(totalSlotMs); if (ddlTargetTable != null) { queryStatisticsPb.setDdlTargetTable(ddlTargetTable.toPb()); queryStatisticsPb.setReferencedTables( Lists.transform(referencedTables, TableId.TO_PB_FUNCTION)); queryStatisticsPb.setStatementType(statementType.toString()); queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); queryStatisticsPb.setTimeline(Lists.transform(timeline, TimelineSample.TO_PB_FUNCTION)); queryStatisticsPb.setSchema(schema.toPb());
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getQuery() != null) { this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed(); if (statisticsPb.getQuery().getDdlTargetTable() != null) { this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable()); this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed(); this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows(); this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled(); this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed(); this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed(); this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs(); if (statisticsPb.getQuery().getStatementType() != null) { this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType()); if (statisticsPb.getQuery().getReferencedTables() != null) { this.referencedTables = Lists.transform( statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION); if (statisticsPb.getQuery().getQueryPlan() != null) { this.queryPlan = Lists.transform( statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION); if (statisticsPb.getQuery().getTimeline() != null) { this.timeline =