@ProcessElement public void process(ProcessContext c) throws Exception { String filepattern = c.element(); MatchResult match = FileSystems.match(filepattern, emptyMatchTreatment); LOG.info("Matched {} files for pattern {}", match.metadata().size(), filepattern); for (MatchResult.Metadata metadata : match.metadata()) { c.output(metadata); } } }
private static MatchResult maybeAdjustEmptyMatchResult( String spec, MatchResult res, EmptyMatchTreatment emptyMatchTreatment) throws IOException { if (res.status() == Status.NOT_FOUND || (res.status() == Status.OK && res.metadata().isEmpty())) { boolean notFoundAllowed = emptyMatchTreatment == EmptyMatchTreatment.ALLOW || (hasGlobWildcard(spec) && emptyMatchTreatment == EmptyMatchTreatment.ALLOW_IF_WILDCARD); return notFoundAllowed ? MatchResult.create(Status.OK, Collections.emptyList()) : MatchResult.create( Status.NOT_FOUND, new FileNotFoundException("No files matched spec: " + spec)); } return res; }
/** * Returns the {@link Metadata} for a single file resource. Expects a resource specification * {@code spec} that matches a single result. * * @param spec a resource specification that matches exactly one result. * @return the {@link Metadata} for the specified resource. * @throws FileNotFoundException if the file resource is not found. * @throws IOException in the event of an error in the inner call to {@link #match}, or if the * given spec does not match exactly 1 result. */ public static Metadata matchSingleFileSpec(String spec) throws IOException { List<MatchResult> matches = FileSystems.match(Collections.singletonList(spec)); MatchResult matchResult = Iterables.getOnlyElement(matches); if (matchResult.status() == Status.NOT_FOUND) { throw new FileNotFoundException(String.format("File spec %s not found", spec)); } else if (matchResult.status() != Status.OK) { throw new IOException( String.format("Error matching file spec %s: status %s", spec, matchResult.status())); } else { List<Metadata> metadata = matchResult.metadata(); if (metadata.size() != 1) { throw new IOException( String.format( "Expecting spec %s to match exactly one file, but matched %s: %s", spec, metadata.size(), metadata)); } return metadata.get(0); } }
private MatchResult toMatchResult(StorageObjectOrIOException objectOrException) { @Nullable IOException exception = objectOrException.ioException(); if (exception instanceof FileNotFoundException) { return MatchResult.create(Status.NOT_FOUND, exception); } else if (exception != null) { return MatchResult.create(Status.ERROR, exception); } else { StorageObject object = objectOrException.storageObject(); assert object != null; // fix a warning; guaranteed by StorageObjectOrIOException semantics. return MatchResult.create(Status.OK, ImmutableList.of(toMetadata(object))); } }
private static KV<List<ResourceId>, List<ResourceId>> filterMissingFiles( List<ResourceId> srcResourceIds, List<ResourceId> destResourceIds) throws IOException { validateSrcDestLists(srcResourceIds, destResourceIds); if (srcResourceIds.isEmpty()) { // Short-circuit. return KV.of(Collections.<ResourceId>emptyList(), Collections.<ResourceId>emptyList()); } List<ResourceId> srcToHandle = new ArrayList<>(); List<ResourceId> destToHandle = new ArrayList<>(); List<MatchResult> matchResults = matchResources(srcResourceIds); for (int i = 0; i < matchResults.size(); ++i) { if (!matchResults.get(i).status().equals(Status.NOT_FOUND)) { srcToHandle.add(srcResourceIds.get(i)); destToHandle.add(destResourceIds.get(i)); } } return KV.of(srcToHandle, destToHandle); }
@VisibleForTesting MatchResult matchNonGlobPath(S3ResourceId path) { ObjectMetadata s3Metadata; try { s3Metadata = getObjectMetadata(path); } catch (AmazonClientException e) { if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 404) { return MatchResult.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()); } return MatchResult.create(MatchResult.Status.ERROR, new IOException(e)); } return MatchResult.create( MatchResult.Status.OK, ImmutableList.of( createBeamMetadata( path.withSize(s3Metadata.getContentLength()), Strings.nullToEmpty(s3Metadata.getContentEncoding())))); }
@Test public void testMatchForNonExistentFile() throws Exception { temporaryFolder.newFile("aa"); List<MatchResult> matchResults = localFileSystem.match( ImmutableList.of(temporaryFolder.getRoot().toPath().resolve("a").toString())); assertEquals(1, matchResults.size()); assertEquals(MatchResult.Status.NOT_FOUND, matchResults.get(0).status()); }
Collection<Metadata> files = FileSystems.match(filePattern).metadata(); LOG.debug( "Found file(s) {} by matching the path: {}",
return MatchResult.create(Status.OK, ImmutableList.of(toMetadata(file))); return MatchResult.create(Status.NOT_FOUND, Collections.emptyList()); return MatchResult.create( Status.NOT_FOUND, new FileNotFoundException(String.format("No files found for spec: %s.", spec))); } else { return MatchResult.create(Status.OK, result);
@Test public void testMatchPatternNone() throws Exception { temporaryFolder.newFile("a"); temporaryFolder.newFile("aa"); temporaryFolder.newFile("ab"); List<MatchResult> matchResults = matchGlobWithPathPrefix(temporaryFolder.getRoot().toPath().resolve("b"), "*"); assertEquals(1, matchResults.size()); assertEquals(MatchResult.Status.NOT_FOUND, matchResults.get(0).status()); }
@Override protected List<MatchResult> match(List<String> specs) { ImmutableList.Builder<MatchResult> resultsBuilder = ImmutableList.builder(); for (String spec : specs) { try { FileStatus[] fileStatuses = fileSystem.globStatus(new Path(spec)); if (fileStatuses == null) { resultsBuilder.add(MatchResult.create(Status.NOT_FOUND, Collections.emptyList())); continue; } List<Metadata> metadata = new ArrayList<>(); for (FileStatus fileStatus : fileStatuses) { if (fileStatus.isFile()) { URI uri = dropEmptyAuthority(fileStatus.getPath().toUri().toString()); metadata.add( Metadata.builder() .setResourceId(new HadoopResourceId(uri)) .setIsReadSeekEfficient(true) .setSizeBytes(fileStatus.getLen()) .build()); } } resultsBuilder.add(MatchResult.create(Status.OK, metadata)); } catch (IOException e) { resultsBuilder.add(MatchResult.create(Status.ERROR, e)); } } return resultsBuilder.build(); }
resourceIdsToDelete = FluentIterable.from(matchResources(Lists.newArrayList(resourceIds))) .filter(matchResult -> !matchResult.status().equals(Status.NOT_FOUND)) .transformAndConcat( new Function<MatchResult, Iterable<Metadata>>() {
@Override public final BoundedReader<T> createReader(PipelineOptions options) throws IOException { // Validate the current source prior to creating a reader for it. this.validate(); String fileOrPattern = fileOrPatternSpec.get(); if (mode == Mode.FILEPATTERN) { long startTime = System.currentTimeMillis(); List<Metadata> fileMetadata = FileSystems.match(fileOrPattern, emptyMatchTreatment).metadata(); LOG.info("Matched {} files for pattern {}", fileMetadata.size(), fileOrPattern); List<FileBasedReader<T>> fileReaders = new ArrayList<>(); for (Metadata metadata : fileMetadata) { long endOffset = metadata.sizeBytes(); fileReaders.add( createForSubrangeOfFile(metadata, 0, endOffset).createSingleFileReader(options)); } LOG.debug( "Creating a reader for file pattern {} took {} ms", fileOrPattern, System.currentTimeMillis() - startTime); if (fileReaders.size() == 1) { return fileReaders.get(0); } return new FilePatternReader(this, fileReaders); } else { return createSingleFileReader(options); } }
results.add(MatchResult.create(MatchResult.Status.ERROR, expandedGlob.getException())); results.add(MatchResult.create(MatchResult.Status.NOT_FOUND, exception)); } else { results.add(MatchResult.create(MatchResult.Status.ERROR, exception)); results.add(MatchResult.create(MatchResult.Status.OK, metadatas));
ImmutableList.of("gs://testbucket/testdirectory/dir2name/"), contains(toFilenames(matchResults.get(1)).toArray())); assertEquals(Status.NOT_FOUND, matchResults.get(2).status()); assertEquals(Status.ERROR, matchResults.get(3).status()); assertThat( ImmutableList.of("gs://testbucket/testdirectory/file4name"),
@Override public Watch.Growth.PollResult<MatchResult.Metadata> apply(String element, Context c) throws Exception { Instant now = Instant.now(); return Watch.Growth.PollResult.incomplete( now, FileSystems.match(element, EmptyMatchTreatment.ALLOW).metadata()) .withWatermark(now); } }
private List<MatchResult> matchGlobs(List<GcsPath> globs) { // TODO: Executes in parallel, address https://issues.apache.org/jira/browse/BEAM-1503. return FluentIterable.from(globs) .transform( gcsPath -> { try { return expand(gcsPath); } catch (IOException e) { return MatchResult.create(Status.ERROR, e); } }) .toList(); }
List<MatchResult> matchResults = gcsFileSystem.match(specs); assertEquals(3, matchResults.size()); assertEquals(Status.OK, matchResults.get(0).status()); assertThat( ImmutableList.of( "gs://testbucket/testdirectory/file3name"), contains(toFilenames(matchResults.get(0)).toArray())); assertEquals(Status.NOT_FOUND, matchResults.get(1).status()); assertEquals(Status.OK, matchResults.get(2).status()); assertThat( ImmutableList.of("gs://testbucket/testdirectory/otherfile"),