!Strings.isNullOrEmpty(tempLocation), "BigQueryIO.Read needs a GCS temp location to store temp files."); if (getBigQueryServices() == null) { try { GcsPath.fromUri(tempLocation); DatasetService datasetService = getBigQueryServices().getDatasetService(bqOptions); BigQueryHelpers.verifyDatasetPresence(datasetService, table.get()); BigQueryHelpers.verifyTablePresence(datasetService, table.get()); checkArgument( getQuery().isAccessible(), "Cannot call validate if query is dynamically set."); JobService jobService = getBigQueryServices().getJobService(bqOptions); try { jobService.dryRunQuery(
private BigQuerySourceBase<T> createSource(String jobUuid, Coder<T> coder) { BigQuerySourceBase<T> source; if (getQuery() == null) { source = BigQueryTableSource.create( jobUuid, getTableProvider(), getBigQueryServices(), coder, getParseFn()); } else { source = BigQueryQuerySource.create( jobUuid, getQuery(), getFlattenResults(), getUseLegacySql(), getBigQueryServices(), coder, getParseFn(), MoreObjects.firstNonNull(getQueryPriority(), QueryPriority.BATCH), getQueryLocation()); } return source; }
@Override void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception { PipelineOptions options = c.getPipelineOptions(); BigQueryOptions bqOptions = options.as(BigQueryOptions.class); String jobUuid = c.getJobId(); final String extractDestinationDir = resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", jobUuid); final String executingProject = bqOptions.getProject(); JobReference jobRef = new JobReference() .setProjectId(executingProject) .setJobId(getExtractJobId(createJobIdToken(bqOptions.getJobName(), jobUuid))); Job extractJob = getBigQueryServices().getJobService(bqOptions).getJob(jobRef); if (extractJob != null) { List<ResourceId> extractFiles = getExtractFilePaths(extractDestinationDir, extractJob); if (extractFiles != null && !extractFiles.isEmpty()) { FileSystems.delete( extractFiles, MoveOptions.StandardMoveOptions.IGNORE_MISSING_FILES); } } } };