private String formatRows(int totalNumRows) { StringBuilder samples = new StringBuilder(); List<TableRow> rows = response.getRows(); for (int i = 0; i < totalNumRows && i < rows.size(); i++) { samples.append(String.format("%n\t\t")); for (TableCell field : rows.get(i).getF()) { samples.append(String.format("%-10s", field.getV())); } } if (rows.size() > totalNumRows) { samples.append(String.format("%n\t\t...")); } return samples.toString(); } }
private QueryResponse getTypedTableRows(QueryResponse response) { List<TableRow> rows = response.getRows(); TableSchema schema = response.getSchema(); response.setRows( rows.stream() .map(r -> getTypedTableRow(schema.getFields(), r)) .collect(Collectors.toList())); return response; }
@Override protected boolean matchesSafely(PipelineResult pipelineResult) { LOG.info("Verifying Bigquery data"); // execute query LOG.debug("Executing query: {}", query); try { response = bigqueryClient.queryWithRetries(query, this.projectId); } catch (IOException | InterruptedException e) { if (e instanceof InterruptedIOException) { Thread.currentThread().interrupt(); } throw new RuntimeException("Failed to fetch BigQuery data.", e); } if (!response.getJobComplete()) { // query job not complete, verification failed return false; } else { // compute checksum actualChecksum = generateHash(response.getRows()); LOG.debug("Generated a SHA1 checksum based on queried data: {}", actualChecksum); return expectedChecksum.equals(actualChecksum); } }
private void verifyLegacyQueryRes() throws Exception { LOG.info("Starting verifyLegacyQueryRes in outputTable {}", outputTable); List<String> legacyQueryExpectedRes = ImmutableList.of("apple", "orange"); QueryResponse response = bqClient.queryWithRetries(String.format("SELECT fruit from [%s];", outputTable), project); LOG.info("Finished to query result table {}", this.outputTable); List<String> tableResult = response .getRows() .stream() .flatMap(row -> row.getF().stream().map(cell -> cell.getV().toString())) .sorted() .collect(Collectors.toList()); assertEquals(legacyQueryExpectedRes, tableResult); }
List<String> tableResult = response .getRows() .stream() .map(
@Test public void testE2ETrafficMaxLaneFlow() throws Exception { this.options.setBigQuerySchema(FormatMaxesFn.getSchema()); this.options.setProject(this.projectId); this.options.setBigQueryDataset(this.outputDatasetId); this.options.setBigQueryTable(this.outputTable); TrafficMaxLaneFlow.runTrafficMaxLaneFlow(this.options); QueryResponse response = this.bqClient.queryWithRetries( String.format( "SELECT count(*) as total FROM [%s:%s.%s]", this.projectId, this.outputDatasetId, this.outputTable), this.projectId); String res = response.getRows().get(0).getF().get(0).getV().toString(); assertEquals("9763", res); } }
@Test public void testE2ETrafficRoutes() throws Exception { this.options.setBigQuerySchema(FormatStatsFn.getSchema()); this.options.setProject(this.projectId); this.options.setBigQueryDataset(this.outputDatasetId); this.options.setBigQueryTable(this.outputTable); TrafficRoutes.runTrafficRoutes(options); QueryResponse response = this.bqClient.queryWithRetries( String.format( "SELECT count(*) as total FROM [%s:%s.%s]", this.projectId, this.outputDatasetId, this.outputTable), this.projectId); String res = response.getRows().get(0).getF().get(0).getV().toString(); assertEquals("27", res); } }
builder.jobId(completeJobId); builder.jobCompleted(results.getJobComplete()); List<TableRow> rowsPb = results.getRows(); if (results.getJobComplete()) { builder.jobCompleted(true);
builder.jobId(completeJobId); builder.jobCompleted(results.getJobComplete()); List<TableRow> rowsPb = results.getRows(); if (results.getJobComplete()) { builder.jobCompleted(true);
bigQueryClient .queryWithRetries(options.getInput(), bigQueryOptions.getProject(), true) .getRows() .get(0);