@Test public void testConstructionNumericOverflow() { final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), Integer.MAX_VALUE, // unlimited total Integer.MAX_VALUE, // limited outgoing Integer.MAX_VALUE, // unlimited incoming Long.MAX_VALUE - 1, // long timeout, close to overflow Long.MAX_VALUE - 1); // long timeout, close to overflow assertEquals(Integer.MAX_VALUE, limitedFs.getMaxNumOpenStreamsTotal()); assertEquals(Integer.MAX_VALUE, limitedFs.getMaxNumOpenOutputStreams()); assertEquals(Integer.MAX_VALUE, limitedFs.getMaxNumOpenInputStreams()); assertTrue(limitedFs.getStreamOpenTimeout() > 0); assertTrue(limitedFs.getStreamInactivityTimeout() > 0); }
!hasAvailability(totalLimit, outputLimit, inputLimit)) { !hasAvailability(totalLimit, outputLimit, inputLimit)) { if (!(closeInactiveStream(openOutputStreams, now) || closeInactiveStream(openInputStreams, now))) { if (timeLeft <= 0 && !hasAvailability(totalLimit, outputLimit, inputLimit)) { throw new IOException(String.format( "Timeout while waiting for an available stream/connection. " + "limits: total=%d, input=%d, output=%d ; Open: input=%d, output=%d ; timeout: %d ms", maxNumOpenStreamsTotal, maxNumOpenInputStreams, maxNumOpenOutputStreams, numReservedInputStreams, numReservedOutputStreams, getStreamOpenTimeout()));
@Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { return createInputStream(() -> originalFs.open(f, bufferSize)); }
@Test public void testFailingStreamsUnregister() throws Exception { final LimitedConnectionsFileSystem fs = new LimitedConnectionsFileSystem(new FailFs(), 1); assertEquals(0, fs.getNumberOfOpenInputStreams()); assertEquals(0, fs.getNumberOfOpenOutputStreams()); assertEquals(0, fs.getTotalNumberOfOpenStreams()); try { fs.open(new Path(tempFolder.newFile().toURI())); fail("this is expected to fail with an exception"); } catch (IOException e) { // expected } try { fs.create(new Path(tempFolder.newFile().toURI()), WriteMode.NO_OVERWRITE); fail("this is expected to fail with an exception"); } catch (IOException e) { // expected } assertEquals(0, fs.getNumberOfOpenInputStreams()); assertEquals(0, fs.getNumberOfOpenOutputStreams()); assertEquals(0, fs.getTotalNumberOfOpenStreams()); }
final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 1000); final Random rnd = new Random(); lfs.isDistributedFS(); verify(fs).isDistributedFS(); lfs.getWorkingDirectory(); verify(fs).isDistributedFS(); lfs.getHomeDirectory(); verify(fs).getHomeDirectory(); lfs.getUri(); verify(fs).getUri(); lfs.getFileStatus(path); verify(fs).getFileStatus(path); int pos = rnd.nextInt(); int len = rnd.nextInt(); lfs.getFileBlockLocations(path, pos, len); verify(fs).getFileBlockLocations(path, pos, len); lfs.open(path, bufferSize); verify(fs).open(path, bufferSize); lfs.open(path); verify(fs).open(path);
@Override public FileSystem create(URI fsUri) throws IOException { FileSystem original = factory.create(fsUri); return new LimitedConnectionsFileSystem(original, settings.limitTotal, settings.limitOutput, settings.limitInput, settings.streamOpenTimeout, settings.streamInactivityTimeout); }
final int maxConcurrentOpen = 2; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), while (limitedFs.getTotalNumberOfOpenStreams() < maxConcurrentOpen) { Thread.sleep(1); limitedFs.create(new Path(tempFolder.newFile().toURI()), WriteMode.OVERWRITE); fail("this should have timed out");
final int maxConcurrentOpen = 2; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), while (limitedFs.getTotalNumberOfOpenStreams() < maxConcurrentOpen) { Thread.sleep(1); createRandomContents(file, rnd); try { limitedFs.open(new Path(file.toURI())); fail("this should have timed out");
@Test public void testDelegateOutStreamMethods() throws IOException { // mock the output stream final FSDataOutputStream mockOut = mock(FSDataOutputStream.class); final long outPos = 46651L; when(mockOut.getPos()).thenReturn(outPos); final FileSystem fs = mock(FileSystem.class); when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mockOut); final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100); final FSDataOutputStream out = lfs.create(mock(Path.class), WriteMode.OVERWRITE); // validate the output stream out.write(77); verify(mockOut).write(77); { byte[] bytes = new byte[1786]; out.write(bytes, 100, 111); verify(mockOut).write(bytes, 100, 111); } assertEquals(outPos, out.getPos()); out.flush(); verify(mockOut).flush(); out.sync(); verify(mockOut).sync(); out.close(); verify(mockOut).close(); }
when(fs.open(any(Path.class))).thenReturn(mockIn); final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100); final FSDataInputStream in = lfs.open(mock(Path.class));
private FSDataOutputStream createOutputStream( final SupplierWithException<FSDataOutputStream, IOException> streamOpener) throws IOException { final SupplierWithException<OutStream, IOException> wrappedStreamOpener = () -> new OutStream(streamOpener.get(), this); return createStream(wrappedStreamOpener, openOutputStreams, true); }
@Override @Deprecated @SuppressWarnings("deprecation") public FSDataOutputStream create( Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { return createOutputStream(() -> originalFs.create(f, overwrite, bufferSize, replication, blockSize)); }
private static FileSystem limitIfConfigured(HadoopFileSystem fs, String scheme, Configuration config) { final ConnectionLimitingSettings limitSettings = ConnectionLimitingSettings.fromConfig(config, scheme); // decorate only if any limit is configured if (limitSettings == null) { // no limit configured return fs; } else { return new LimitedConnectionsFileSystem( fs, limitSettings.limitTotal, limitSettings.limitOutput, limitSettings.limitInput, limitSettings.streamOpenTimeout, limitSettings.streamInactivityTimeout); } } }
final LimitedConnectionsFileSystem fs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), 1, 0L, 1000L); try (FSDataOutputStream out = fs.create(new Path(tempFolder.newFile().toURI()), WriteMode.OVERWRITE)) {
createRandomContents(file, new Random(), 50); final LimitedConnectionsFileSystem fs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), 1, 0L, 1000L); try (FSDataInputStream in = fs.open(new Path(file.toURI()))) {
private FSDataInputStream createInputStream( final SupplierWithException<FSDataInputStream, IOException> streamOpener) throws IOException { final SupplierWithException<InStream, IOException> wrappedStreamOpener = () -> new InStream(streamOpener.get(), this); return createStream(wrappedStreamOpener, openInputStreams, false); }
@Override @Deprecated @SuppressWarnings("deprecation") public FSDataOutputStream create( Path f, boolean overwrite, int bufferSize, short replication, long blockSize) throws IOException { return createOutputStream(() -> originalFs.create(f, overwrite, bufferSize, replication, blockSize)); }
assertEquals(40, limitedFs.getMaxNumOpenStreamsTotal()); assertEquals(39, limitedFs.getMaxNumOpenInputStreams()); assertEquals(38, limitedFs.getMaxNumOpenOutputStreams()); assertEquals(23456, limitedFs.getStreamOpenTimeout()); assertEquals(34567, limitedFs.getStreamInactivityTimeout());
@Test public void testLimitingMixedStreams() throws Exception { final int maxConcurrentOpen = 2; final int numThreads = 61; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), maxConcurrentOpen); // limited total final Random rnd = new Random(); final CheckedThread[] threads = new CheckedThread[numThreads]; for (int i = 0; i < numThreads; i++) { File file = tempFolder.newFile(); Path path = new Path(file.toURI()); if (rnd.nextBoolean()) { // reader thread createRandomContents(file, rnd); threads[i] = new ReaderThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen); } else { threads[i] = new WriterThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen); } } for (CheckedThread t : threads) { t.start(); } for (CheckedThread t : threads) { t.sync(); } }
!hasAvailability(totalLimit, outputLimit, inputLimit)) { !hasAvailability(totalLimit, outputLimit, inputLimit)) { if (!(closeInactiveStream(openOutputStreams, now) || closeInactiveStream(openInputStreams, now))) { if (timeLeft <= 0 && !hasAvailability(totalLimit, outputLimit, inputLimit)) { throw new IOException(String.format( "Timeout while waiting for an available stream/connection. " + "limits: total=%d, input=%d, output=%d ; Open: input=%d, output=%d ; timeout: %d ms", maxNumOpenStreamsTotal, maxNumOpenInputStreams, maxNumOpenOutputStreams, numReservedInputStreams, numReservedOutputStreams, getStreamOpenTimeout()));