@Override public String create(PipelineOptions options) { if (options.getTempLocation() != null) { return GcsPath.fromUri(options.getTempLocation()).resolve("deduped.txt").toString(); } else { throw new IllegalArgumentException("Must specify --output or --tempLocation"); } } }
@Override public void validate(PipelineOptions options) { // We will use a BigQuery load job -- validate the temp location. String tempLocation; if (customGcsTempLocation == null) { tempLocation = options.getTempLocation(); } else { if (!customGcsTempLocation.isAccessible()) { // Can't perform verification in this case. return; } tempLocation = customGcsTempLocation.get(); } checkArgument( !Strings.isNullOrEmpty(tempLocation), "BigQueryIO.Write needs a GCS temp location to store temp files."); if (bigQueryServices == null) { try { GcsPath.fromUri(tempLocation); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( String.format( "BigQuery temp location expected a valid 'gs://' path, but was given '%s'", tempLocation), e); } } }
@After public void tearDown() throws IOException { testNumFiles(new File(options.getTempLocation()), 0); }
@ProcessElement public void getTempFilePrefix(ProcessContext c) { String tempLocationRoot; if (customGcsTempLocation != null) { tempLocationRoot = customGcsTempLocation.get(); } else { tempLocationRoot = c.getPipelineOptions().getTempLocation(); } String tempLocation = resolveTempLocation( tempLocationRoot, "BigQueryWriteTemp", c.sideInput(jobIdView)); LOG.info( "Writing BigQuery temporary files to {} before loading them.", tempLocation); c.output(tempLocation); } })
@Test public void testRemoveTemporaryFiles() throws Exception { int numFiles = 10; List<String> fileNames = Lists.newArrayList(); String tempFilePrefix = options.getTempLocation() + "/"; for (int i = 0; i < numFiles; ++i) { TableRowWriter writer = new TableRowWriter(tempFilePrefix); writer.close(); fileNames.add(writer.getResult().resourceId.toString()); } fileNames.add(tempFilePrefix + String.format("files%05d", numFiles)); File tempDir = new File(options.getTempLocation()); testNumFiles(tempDir, 10); WriteTables.removeTemporaryFiles(fileNames); }
@Override @Nullable public String create(PipelineOptions options) { String tempLocation = options.getTempLocation(); if (isNullOrEmpty(tempLocation)) { tempLocation = tryCreateDefaultBucket( options, newCloudResourceManagerClient(options.as(CloudResourceManagerOptions.class)) .build()); options.setTempLocation(tempLocation); } else { try { PathValidator validator = options.as(GcsOptions.class).getPathValidator(); validator.validateOutputFilePrefixSupported(tempLocation); } catch (Exception e) { throw new IllegalArgumentException( String.format( "Error constructing default value for gcpTempLocation: tempLocation is not" + " a valid GCS path, %s. ", tempLocation), e); } } return tempLocation; }