/** * Returns the FileSystem that owns this Path. * * @return the FileSystem that owns this Path * @throws IOException * thrown if the file system could not be retrieved */ public FileSystem getFileSystem() throws IOException { return FileSystem.get(this.toUri()); }
@Override public void run(SourceContext<Tuple3<String, Long, Long>> ctx) throws Exception { FileSystem fileSystem = FileSystem.get(new URI(path)); while (isRunning) { List<String> files = listNewFiles(fileSystem); for (String filePath : files) { if (watchType == WatchType.ONLY_NEW_FILES || watchType == WatchType.REPROCESS_WITH_APPENDED) { ctx.collect(new Tuple3<String, Long, Long>(filePath, 0L, -1L)); offsetOfFiles.put(filePath, -1L); } else if (watchType == WatchType.PROCESS_ONLY_APPENDED) { long offset = 0; long fileSize = fileSystem.getFileStatus(new Path(filePath)).getLen(); if (offsetOfFiles.containsKey(filePath)) { offset = offsetOfFiles.get(filePath); } ctx.collect(new Tuple3<String, Long, Long>(filePath, offset, fileSize)); offsetOfFiles.put(filePath, fileSize); LOG.info("File processed: {}, {}, {}", filePath, offset, fileSize); } } Thread.sleep(interval); } }
@Test public void testDefaultsToLocal() throws Exception { URI justPath = new URI(tempFolder.newFile().toURI().getPath()); assertNull(justPath.getScheme()); FileSystem fs = FileSystem.get(justPath); assertEquals("file", fs.getUri().getScheme()); }
final FileSystem fs = FileSystem.get(filePath.toUri());
@Override public void flatMap(Tuple3<String, Long, Long> value, Collector<String> out) throws Exception { FSDataInputStream stream = FileSystem.get(new URI(value.f0)).open(new Path(value.f0)); stream.seek(value.f1); BufferedReader reader = new BufferedReader(new InputStreamReader(stream)); String line; try { while ((line = reader.readLine()) != null && (value.f2 == -1L || stream.getPos() <= value.f2)) { out.collect(line); } } finally { reader.close(); } } }
@Test public void testExplicitlySetToLocal() throws Exception { final Configuration conf = new Configuration(); conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, LocalFileSystem.getLocalFsURI().toString()); FileSystem.initialize(conf); URI justPath = new URI(tempFolder.newFile().toURI().getPath()); assertNull(justPath.getScheme()); FileSystem fs = FileSystem.get(justPath); assertEquals("file", fs.getUri().getScheme()); }
final FileSystem fs = FileSystem.get(filePath.toUri());
@Override public void tryCleanupOnError() { if (this.fileCreated) { this.fileCreated = false; try { close(); } catch (IOException e) { LOG.error("Could not properly close FileOutputFormat.", e); } try { FileSystem.get(this.actualFilePath.toUri()).delete(actualFilePath, false); } catch (FileNotFoundException e) { // ignore, may not be visible yet or may be already removed } catch (Throwable t) { LOG.error("Could not remove the incomplete file " + actualFilePath + '.', t); } } } }
@Test public void testExplicitlyPathTakesPrecedence() throws Exception { final Configuration conf = new Configuration(); conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/"); FileSystem.initialize(conf); URI pathAndScheme = tempFolder.newFile().toURI(); assertNotNull(pathAndScheme.getScheme()); FileSystem fs = FileSystem.get(pathAndScheme); assertEquals("file", fs.getUri().getScheme()); } }
private static FileSystem getFileSystemWithoutSafetyNet(final String uri) throws URISyntaxException, IOException { final FileSystem fileSystem = FileSystem.get(new URI(uri)); if (fileSystem instanceof WrappingProxy) { //noinspection unchecked return WrappingProxyUtil.stripProxy((WrappingProxy<FileSystem>) fileSystem); } return fileSystem; }
@Override public void run(SourceFunction.SourceContext<TimestampedFileInputSplit> context) throws Exception { Path p = new Path(path); FileSystem fileSystem = FileSystem.get(p.toUri()); if (!fileSystem.exists(p)) { throw new FileNotFoundException("The provided file path " + path + " does not exist.");
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
protected FileBaseStatistics getFileStats(FileBaseStatistics cachedStats, Path[] filePaths, ArrayList<FileStatus> files) throws IOException { long totalLength = 0; long latestModTime = 0; for (Path path : filePaths) { final FileSystem fs = FileSystem.get(path.toUri()); final FileBaseStatistics stats = getFileStats(cachedStats, path, fs, files); if (stats.getTotalInputSize() == BaseStatistics.SIZE_UNKNOWN) { totalLength = BaseStatistics.SIZE_UNKNOWN; } else if (totalLength != BaseStatistics.SIZE_UNKNOWN) { totalLength += stats.getTotalInputSize(); } latestModTime = Math.max(latestModTime, stats.getLastModificationTime()); } // check whether the cached statistics are still valid, if we have any if (cachedStats != null && latestModTime <= cachedStats.getLastModificationTime()) { return cachedStats; } return new FileBaseStatistics(latestModTime, totalLength, BaseStatistics.AVG_RECORD_BYTES_UNKNOWN); }
private static TestRecoverableWriter getRecoverableWriter(Path path) { try { final FileSystem fs = FileSystem.get(path.toUri()); if (!(fs instanceof LocalFileSystem)) { fail("Expected Local FS but got a " + fs.getClass().getName() + " for path: " + path); } return new TestRecoverableWriter((LocalFileSystem) fs); } catch (IOException e) { fail(); } return null; }
@Test public void testExplicitlySetToOther() throws Exception { final Configuration conf = new Configuration(); conf.setString(CoreOptions.DEFAULT_FILESYSTEM_SCHEME, "otherFS://localhost:1234/"); FileSystem.initialize(conf); URI justPath = new URI(tempFolder.newFile().toURI().getPath()); assertNull(justPath.getScheme()); try { FileSystem.get(justPath); fail("should have failed with an exception"); } catch (UnsupportedFileSystemSchemeException e) { assertTrue(e.getMessage().contains("otherFS")); } }
this.fsWriter = FileSystem.get(basePath.toUri()).createRecoverableWriter(); } catch (IOException e) { LOG.error("Unable to create filesystem for path: {}", basePath);
@Test public void testSerializationEmpty() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final RecoverableWriter writer = fs.createRecoverableWriter(); final Path testBucket = new Path(testFolder.getPath(), "test"); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, null, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); Assert.assertNull(recoveredState.getInProgressResumableFile()); Assert.assertTrue(recoveredState.getCommittableFilesPerCheckpoint().isEmpty()); }
FileSystem hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c")); FileSystem ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c")); FileSystem.initialize(config); hdfs = FileSystem.get(URI.create("hdfs://localhost:12345/a/b/c")); ftpfs = FileSystem.get(URI.create("ftp://localhost:12345/a/b/c"));
@Test public void testSerializationOnlyInProgress() throws IOException { final File testFolder = tempFolder.newFolder(); final FileSystem fs = FileSystem.get(testFolder.toURI()); final Path testBucket = new Path(testFolder.getPath(), "test"); final RecoverableWriter writer = fs.createRecoverableWriter(); final RecoverableFsDataOutputStream stream = writer.open(testBucket); stream.write(IN_PROGRESS_CONTENT.getBytes(Charset.forName("UTF-8"))); final RecoverableWriter.ResumeRecoverable current = stream.persist(); final BucketState<String> bucketState = new BucketState<>( "test", testBucket, Long.MAX_VALUE, current, new HashMap<>()); final SimpleVersionedSerializer<BucketState<String>> serializer = new BucketStateSerializer<>( writer.getResumeRecoverableSerializer(), writer.getCommitRecoverableSerializer(), SimpleVersionedStringSerializer.INSTANCE ); final byte[] bytes = SimpleVersionedSerialization.writeVersionAndSerialize(serializer, bucketState); // to simulate that everything is over for file. stream.close(); final BucketState<String> recoveredState = SimpleVersionedSerialization.readVersionAndDeSerialize(serializer, bytes); Assert.assertEquals(testBucket, recoveredState.getBucketPath()); FileStatus[] statuses = fs.listStatus(testBucket.getParent()); Assert.assertEquals(1L, statuses.length); Assert.assertTrue( statuses[0].getPath().getPath().startsWith( (new Path(testBucket.getParent(), ".test.inprogress")).toString()) ); }