JobId completeJobId = JobId.fromPb(results.getJobReference()); builder.jobId(completeJobId); builder.jobCompleted(results.getJobComplete()); List<TableRow> rowsPb = results.getRows(); if (results.getJobComplete()) { builder.jobCompleted(true); QueryResult.Builder resultBuilder = transformQueryResults(completeJobId, rowsPb, results.getPageToken(), options(), ImmutableMap.<BigQueryRpc.Option, Object>of()); resultBuilder.totalBytesProcessed(results.getTotalBytesProcessed()); resultBuilder.cacheHit(results.getCacheHit()); if (results.getSchema() != null) { resultBuilder.schema(Schema.fromPb(results.getSchema())); if (results.getTotalRows() != null) { resultBuilder.totalRows(results.getTotalRows().longValue()); if (results.getErrors() != null) { builder.executionErrors( Lists.transform(results.getErrors(), BigQueryError.FROM_PB_FUNCTION));
@Override public void describeMismatchSafely(PipelineResult pResult, Description description) { String info; if (!response.getJobComplete()) { // query job not complete info = String.format("The query job hasn't completed. Got response: %s", response); } else { // checksum mismatch info = String.format( "was (%s).%n" + "\tTotal number of rows are: %d.%n" + "\tQueried data details:%s", actualChecksum, response.getTotalRows(), formatRows(TOTAL_FORMATTED_ROWS)); } description.appendText(info); }
@Override protected boolean matchesSafely(PipelineResult pipelineResult) { LOG.info("Verifying Bigquery data"); // execute query LOG.debug("Executing query: {}", query); try { response = bigqueryClient.queryWithRetries(query, this.projectId); } catch (IOException | InterruptedException e) { if (e instanceof InterruptedIOException) { Thread.currentThread().interrupt(); } throw new RuntimeException("Failed to fetch BigQuery data.", e); } if (!response.getJobComplete()) { // query job not complete, verification failed return false; } else { // compute checksum actualChecksum = generateHash(response.getRows()); LOG.debug("Generated a SHA1 checksum based on queried data: {}", actualChecksum); return expectedChecksum.equals(actualChecksum); } }
private String formatRows(int totalNumRows) { StringBuilder samples = new StringBuilder(); List<TableRow> rows = response.getRows(); for (int i = 0; i < totalNumRows && i < rows.size(); i++) { samples.append(String.format("%n\t\t")); for (TableCell field : rows.get(i).getF()) { samples.append(String.format("%-10s", field.getV())); } } if (rows.size() > totalNumRows) { samples.append(String.format("%n\t\t...")); } return samples.toString(); } }
private QueryResponse createResponseContainingTestData() { TableCell field1 = new TableCell(); field1.setV("abc"); TableCell field2 = new TableCell(); field2.setV("2"); TableCell field3 = new TableCell(); field3.setV("testing BigQuery matcher."); TableRow row = new TableRow(); row.setF(Lists.newArrayList(field1, field2, field3)); QueryResponse response = new QueryResponse(); response.setJobComplete(true); response.setRows(Lists.newArrayList(row)); response.setTotalRows(BigInteger.ONE); return response; } }
private void verifyLegacyQueryRes() throws Exception { LOG.info("Starting verifyLegacyQueryRes in outputTable {}", outputTable); List<String> legacyQueryExpectedRes = ImmutableList.of("apple", "orange"); QueryResponse response = bqClient.queryWithRetries(String.format("SELECT fruit from [%s];", outputTable), project); LOG.info("Finished to query result table {}", this.outputTable); List<String> tableResult = response .getRows() .stream() .flatMap(row -> row.getF().stream().map(cell -> cell.getV().toString())) .sorted() .collect(Collectors.toList()); assertEquals(legacyQueryExpectedRes, tableResult); }
JobId completeJobId = JobId.fromPb(results.getJobReference()); builder.jobId(completeJobId); builder.jobCompleted(results.getJobComplete()); List<TableRow> rowsPb = results.getRows(); if (results.getJobComplete()) { builder.jobCompleted(true); QueryResult.Builder resultBuilder = transformQueryResults(completeJobId, rowsPb, results.getPageToken(), options(), ImmutableMap.<BigQueryRpc.Option, Object>of()); resultBuilder.totalBytesProcessed(results.getTotalBytesProcessed()); resultBuilder.cacheHit(results.getCacheHit()); if (results.getSchema() != null) { resultBuilder.schema(Schema.fromPb(results.getSchema())); if (results.getTotalRows() != null) { resultBuilder.totalRows(results.getTotalRows().longValue()); if (results.getErrors() != null) { builder.executionErrors( Lists.transform(results.getErrors(), BigQueryError.FROM_PB_FUNCTION));
List<String> tableResult = response .getRows() .stream() .map(
@Test public void testE2ETrafficRoutes() throws Exception { this.options.setBigQuerySchema(FormatStatsFn.getSchema()); this.options.setProject(this.projectId); this.options.setBigQueryDataset(this.outputDatasetId); this.options.setBigQueryTable(this.outputTable); TrafficRoutes.runTrafficRoutes(options); QueryResponse response = this.bqClient.queryWithRetries( String.format( "SELECT count(*) as total FROM [%s:%s.%s]", this.projectId, this.outputDatasetId, this.outputTable), this.projectId); String res = response.getRows().get(0).getF().get(0).getV().toString(); assertEquals("27", res); } }
@Test public void testE2ETrafficMaxLaneFlow() throws Exception { this.options.setBigQuerySchema(FormatMaxesFn.getSchema()); this.options.setProject(this.projectId); this.options.setBigQueryDataset(this.outputDatasetId); this.options.setBigQueryTable(this.outputTable); TrafficMaxLaneFlow.runTrafficMaxLaneFlow(this.options); QueryResponse response = this.bqClient.queryWithRetries( String.format( "SELECT count(*) as total FROM [%s:%s.%s]", this.projectId, this.outputDatasetId, this.outputTable), this.projectId); String res = response.getRows().get(0).getF().get(0).getV().toString(); assertEquals("9763", res); } }
bigQueryClient .queryWithRetries(options.getInput(), bigQueryOptions.getProject(), true) .getRows() .get(0);