com.google.api.services.bigquery.model.JobStatistics toPb() { com.google.api.services.bigquery.model.JobStatistics statistics = new com.google.api.services.bigquery.model.JobStatistics(); statistics.setCreationTime(creationTime); statistics.setEndTime(endTime); statistics.setStartTime(startTime); return statistics; }
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getQuery() != null) { this.billingTier = statisticsPb.getQuery().getBillingTier(); this.cacheHit = statisticsPb.getQuery().getCacheHit(); this.ddlOperationPerformed = statisticsPb.getQuery().getDdlOperationPerformed(); if (statisticsPb.getQuery().getDdlTargetTable() != null) { this.ddlTargetTable = TableId.fromPb(statisticsPb.getQuery().getDdlTargetTable()); this.estimatedBytesProcessed = statisticsPb.getQuery().getEstimatedBytesProcessed(); this.numDmlAffectedRows = statisticsPb.getQuery().getNumDmlAffectedRows(); this.totalBytesBilled = statisticsPb.getQuery().getTotalBytesBilled(); this.totalBytesProcessed = statisticsPb.getQuery().getTotalBytesProcessed(); this.totalPartitionsProcessed = statisticsPb.getQuery().getTotalPartitionsProcessed(); this.totalSlotMs = statisticsPb.getQuery().getTotalSlotMs(); if (statisticsPb.getQuery().getStatementType() != null) { this.statementType = StatementType.valueOf(statisticsPb.getQuery().getStatementType()); if (statisticsPb.getQuery().getReferencedTables() != null) { this.referencedTables = Lists.transform( statisticsPb.getQuery().getReferencedTables(), TableId.FROM_PB_FUNCTION); if (statisticsPb.getQuery().getQueryPlan() != null) { this.queryPlan = Lists.transform( statisticsPb.getQuery().getQueryPlan(), QueryStage.FROM_PB_FUNCTION); if (statisticsPb.getQuery().getTimeline() != null) { this.timeline =
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getExtract() != null) { this.destinationUriFileCounts = statisticsPb.getExtract().getDestinationUriFileCounts(); } }
protected Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { this.creationTime = statisticsPb.getCreationTime(); this.endTime = statisticsPb.getEndTime(); this.startTime = statisticsPb.getStartTime(); }
JobStatistics queryJobStats = new JobStatistics(); JobStatistics2 queryStats = new JobStatistics2(); queryStats.setReferencedTables(ImmutableList.of(dryRunTable)); queryJobStats.setQuery(queryStats); queryJob.setStatus(new JobStatus()).setStatistics(queryJobStats); JobStatistics extractJobStats = new JobStatistics(); JobStatistics4 extractStats = new JobStatistics4(); extractStats.setDestinationUriFileCounts(ImmutableList.of(1L)); extractJobStats.setExtract(extractStats); extractJob.setStatus(new JobStatus()).setStatistics(extractJobStats); "project-id", query, new JobStatistics() .setQuery( new JobStatistics2() .setTotalBytesProcessed(100L)
@Test public void testIncomplete() { // https://github.com/googleapis/google-cloud-java/issues/2357 com.google.api.services.bigquery.model.Job job = new com.google.api.services.bigquery.model.Job() .setStatistics( new com.google.api.services.bigquery.model.JobStatistics() .setCreationTime(1234L) .setStartTime(5678L)); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setCopy(new com.google.api.services.bigquery.model.JobConfigurationTableCopy())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(CopyStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setLoad(new com.google.api.services.bigquery.model.JobConfigurationLoad())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(LoadStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setExtract(new com.google.api.services.bigquery.model.JobConfigurationExtract())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(ExtractStatistics.class); job.setConfiguration( new com.google.api.services.bigquery.model.JobConfiguration() .setQuery(new com.google.api.services.bigquery.model.JobConfigurationQuery())); assertThat(JobStatistics.fromPb(job)).isInstanceOf(QueryStatistics.class); }
@SuppressWarnings("unchecked") static <T extends JobStatistics> T fromPb( com.google.api.services.bigquery.model.JobStatistics statisticPb) { if (statisticPb.getLoad() != null) { return (T) LoadStatistics.fromPb(statisticPb); } else if (statisticPb.getExtract() != null) { return (T) ExtractStatistics.fromPb(statisticPb); } else if (statisticPb.getQuery() != null) { return (T) QueryStatistics.fromPb(statisticPb); } else { return (T) CopyStatistics.fromPb(statisticPb); } } }
private JobStatus runExtractJob(Job job, JobConfigurationExtract extract) throws InterruptedException, IOException { TableReference sourceTable = extract.getSourceTable(); List<TableRow> rows = datasetService.getAllRows( sourceTable.getProjectId(), sourceTable.getDatasetId(), sourceTable.getTableId()); TableSchema schema = datasetService.getTable(sourceTable).getSchema(); List<Long> destinationFileCounts = Lists.newArrayList(); for (String destination : extract.getDestinationUris()) { destinationFileCounts.add(writeRows(sourceTable.getTableId(), rows, schema, destination)); } job.setStatistics( new JobStatistics() .setExtract(new JobStatistics4().setDestinationUriFileCounts(destinationFileCounts))); return new JobStatus().setState("DONE"); }
@Test public void testBigQueryQuerySourceEstimatedSize() throws Exception { List<TableRow> data = ImmutableList.of( new TableRow().set("name", "A").set("number", 10L), new TableRow().set("name", "B").set("number", 11L), new TableRow().set("name", "C").set("number", 12L)); PipelineOptions options = PipelineOptionsFactory.create(); BigQueryOptions bqOptions = options.as(BigQueryOptions.class); bqOptions.setProject("project"); String stepUuid = "testStepUuid"; String query = FakeBigQueryServices.encodeQuery(data); BigQueryQuerySource<TableRow> bqSource = BigQueryQuerySource.create( stepUuid, ValueProvider.StaticValueProvider.of(query), true /* flattenResults */, true /* useLegacySql */, fakeBqServices, TableRowJsonCoder.of(), BigQueryIO.TableRowParser.INSTANCE, QueryPriority.BATCH, null); fakeJobService.expectDryRunQuery( bqOptions.getProject(), query, new JobStatistics().setQuery(new JobStatistics2().setTotalBytesProcessed(100L))); assertEquals(100, bqSource.getEstimatedSizeBytes(bqOptions)); }
@Override com.google.api.services.bigquery.model.JobStatistics toPb() { com.google.api.services.bigquery.model.JobStatistics statisticsPb = super.toPb(); return statisticsPb.setExtract( new JobStatistics4().setDestinationUriFileCounts(destinationUriFileCounts)); }
queryStatisticsPb.setSchema(schema.toPb()); return super.toPb().setQuery(queryStatisticsPb);
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getLoad() != null) { this.inputBytes = statisticsPb.getLoad().getInputFileBytes(); this.inputFiles = statisticsPb.getLoad().getInputFiles(); this.outputBytes = statisticsPb.getLoad().getOutputBytes(); this.outputRows = statisticsPb.getLoad().getOutputRows(); this.badRecords = statisticsPb.getLoad().getBadRecords(); } }
JobStatistics queryJobStats = new JobStatistics(); JobStatistics2 queryStats = new JobStatistics2(); queryStats.setReferencedTables(ImmutableList.of(dryRunTable)); queryJobStats.setQuery(queryStats); queryJob.setStatus(new JobStatus()).setStatistics(queryJobStats); JobStatistics extractJobStats = new JobStatistics(); JobStatistics4 extractStats = new JobStatistics4(); extractStats.setDestinationUriFileCounts(ImmutableList.of(1L)); extractJobStats.setExtract(extractStats); extractJob.setStatus(new JobStatus()).setStatistics(extractJobStats); bqOptions.getProject(), query, new JobStatistics() .setQuery( new JobStatistics2() .setTotalBytesProcessed(100L)
protected Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { this.creationTime = statisticsPb.getCreationTime(); this.endTime = statisticsPb.getEndTime(); this.startTime = statisticsPb.getStartTime(); }
@SuppressWarnings("unchecked") static <T extends JobStatistics> T fromPb( com.google.api.services.bigquery.model.JobStatistics statisticPb) { if (statisticPb.getLoad() != null) { return (T) LoadStatistics.fromPb(statisticPb); } else if (statisticPb.getExtract() != null) { return (T) ExtractStatistics.fromPb(statisticPb); } else if (statisticPb.getQuery() != null) { return (T) QueryStatistics.fromPb(statisticPb); } else { return (T) new Builder(statisticPb).build(); } } }
@Override com.google.api.services.bigquery.model.JobStatistics toPb() { com.google.api.services.bigquery.model.JobStatistics statisticsPb = super.toPb(); return statisticsPb.setExtract( new JobStatistics4().setDestinationUriFileCounts(destinationUriFileCounts)); }
@Override com.google.api.services.bigquery.model.JobStatistics toPb() { JobStatistics2 queryStatisticsPb = new JobStatistics2(); queryStatisticsPb.setBillingTier(billingTier); queryStatisticsPb.setCacheHit(cacheHit); queryStatisticsPb.setTotalBytesBilled(totalBytesBilled); queryStatisticsPb.setTotalBytesProcessed(totalBytesProcessed); if (queryPlan != null) { queryStatisticsPb.setQueryPlan(Lists.transform(queryPlan, QueryStage.TO_PB_FUNCTION)); } return super.toPb().setQuery(queryStatisticsPb); }
private Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { super(statisticsPb); if (statisticsPb.getLoad() != null) { this.inputBytes = statisticsPb.getLoad().getInputFileBytes(); this.inputFiles = statisticsPb.getLoad().getInputFiles(); this.outputBytes = statisticsPb.getLoad().getOutputBytes(); this.outputRows = statisticsPb.getLoad().getOutputRows(); this.badRecords = statisticsPb.getLoad().getBadRecords(); } }
com.google.api.services.bigquery.model.JobStatistics toPb() { com.google.api.services.bigquery.model.JobStatistics statistics = new com.google.api.services.bigquery.model.JobStatistics(); statistics.setCreationTime(creationTime); statistics.setEndTime(endTime); statistics.setStartTime(startTime); return statistics; }
protected Builder(com.google.api.services.bigquery.model.JobStatistics statisticsPb) { this.creationTime = statisticsPb.getCreationTime(); this.endTime = statisticsPb.getEndTime(); this.startTime = statisticsPb.getStartTime(); }