@Override public FileSystem create(URI fsUri) throws IOException { FileSystem original = factory.create(fsUri); return new LimitedConnectionsFileSystem(original, settings.limitTotal, settings.limitOutput, settings.limitInput, settings.streamOpenTimeout, settings.streamInactivityTimeout); }
private static FileSystem limitIfConfigured(HadoopFileSystem fs, String scheme, Configuration config) { final ConnectionLimitingSettings limitSettings = ConnectionLimitingSettings.fromConfig(config, scheme); // decorate only if any limit is configured if (limitSettings == null) { // no limit configured return fs; } else { return new LimitedConnectionsFileSystem( fs, limitSettings.limitTotal, limitSettings.limitOutput, limitSettings.limitInput, limitSettings.streamOpenTimeout, limitSettings.streamInactivityTimeout); } } }
@Test public void testDelegateOutStreamMethods() throws IOException { // mock the output stream final FSDataOutputStream mockOut = mock(FSDataOutputStream.class); final long outPos = 46651L; when(mockOut.getPos()).thenReturn(outPos); final FileSystem fs = mock(FileSystem.class); when(fs.create(any(Path.class), any(WriteMode.class))).thenReturn(mockOut); final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100); final FSDataOutputStream out = lfs.create(mock(Path.class), WriteMode.OVERWRITE); // validate the output stream out.write(77); verify(mockOut).write(77); { byte[] bytes = new byte[1786]; out.write(bytes, 100, 111); verify(mockOut).write(bytes, 100, 111); } assertEquals(outPos, out.getPos()); out.flush(); verify(mockOut).flush(); out.sync(); verify(mockOut).sync(); out.close(); verify(mockOut).close(); }
when(fs.open(any(Path.class))).thenReturn(mockIn); final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 100); final FSDataInputStream in = lfs.open(mock(Path.class));
when(fs.create(any(Path.class), anyBoolean(), anyInt(), anyShort(), anyLong())).thenReturn(mock(FSDataOutputStream.class)); final LimitedConnectionsFileSystem lfs = new LimitedConnectionsFileSystem(fs, 1000); final Random rnd = new Random();
@Test public void testLimitingMixedStreams() throws Exception { final int maxConcurrentOpen = 2; final int numThreads = 61; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), maxConcurrentOpen); // limited total final Random rnd = new Random(); final CheckedThread[] threads = new CheckedThread[numThreads]; for (int i = 0; i < numThreads; i++) { File file = tempFolder.newFile(); Path path = new Path(file.toURI()); if (rnd.nextBoolean()) { // reader thread createRandomContents(file, rnd); threads[i] = new ReaderThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen); } else { threads[i] = new WriterThread(limitedFs, path, Integer.MAX_VALUE, maxConcurrentOpen); } } for (CheckedThread t : threads) { t.start(); } for (CheckedThread t : threads) { t.sync(); } }
@Test public void testLimitingInputStreams() throws Exception { final int maxConcurrentOpen = 2; final int numThreads = 61; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), Integer.MAX_VALUE, // unlimited total Integer.MAX_VALUE, // unlimited outgoing maxConcurrentOpen, // limited incoming 0, 0); final Random rnd = new Random(); final ReaderThread[] threads = new ReaderThread[numThreads]; for (int i = 0; i < numThreads; i++) { File file = tempFolder.newFile(); createRandomContents(file, rnd); Path path = new Path(file.toURI()); threads[i] = new ReaderThread(limitedFs, path, maxConcurrentOpen, Integer.MAX_VALUE); } for (ReaderThread t : threads) { t.start(); } for (ReaderThread t : threads) { t.sync(); } }
createRandomContents(file, new Random(), 50); final LimitedConnectionsFileSystem fs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), 1, 0L, 1000L);
final LimitedConnectionsFileSystem fs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), 1, 0L, 1000L);
@Test public void testLimitingOutputStreams() throws Exception { final int maxConcurrentOpen = 2; final int numThreads = 61; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), Integer.MAX_VALUE, // unlimited total maxConcurrentOpen, // limited outgoing Integer.MAX_VALUE, // unlimited incoming 0, 0); final WriterThread[] threads = new WriterThread[numThreads]; for (int i = 0; i < numThreads; i++) { Path path = new Path(tempFolder.newFile().toURI()); threads[i] = new WriterThread(limitedFs, path, maxConcurrentOpen, Integer.MAX_VALUE); } for (WriterThread t : threads) { t.start(); } for (WriterThread t : threads) { t.sync(); } }
final int maxConcurrentOpen = 2; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(),
final int maxConcurrentOpen = 2; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(),
final int numThreads = 20; final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(),
final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(),
final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(),
@Test public void testConstructionNumericOverflow() { final LimitedConnectionsFileSystem limitedFs = new LimitedConnectionsFileSystem( LocalFileSystem.getSharedInstance(), Integer.MAX_VALUE, // unlimited total Integer.MAX_VALUE, // limited outgoing Integer.MAX_VALUE, // unlimited incoming Long.MAX_VALUE - 1, // long timeout, close to overflow Long.MAX_VALUE - 1); // long timeout, close to overflow assertEquals(Integer.MAX_VALUE, limitedFs.getMaxNumOpenStreamsTotal()); assertEquals(Integer.MAX_VALUE, limitedFs.getMaxNumOpenOutputStreams()); assertEquals(Integer.MAX_VALUE, limitedFs.getMaxNumOpenInputStreams()); assertTrue(limitedFs.getStreamOpenTimeout() > 0); assertTrue(limitedFs.getStreamInactivityTimeout() > 0); }
@Test public void testFailingStreamsUnregister() throws Exception { final LimitedConnectionsFileSystem fs = new LimitedConnectionsFileSystem(new FailFs(), 1); assertEquals(0, fs.getNumberOfOpenInputStreams()); assertEquals(0, fs.getNumberOfOpenOutputStreams()); assertEquals(0, fs.getTotalNumberOfOpenStreams()); try { fs.open(new Path(tempFolder.newFile().toURI())); fail("this is expected to fail with an exception"); } catch (IOException e) { // expected } try { fs.create(new Path(tempFolder.newFile().toURI()), WriteMode.NO_OVERWRITE); fail("this is expected to fail with an exception"); } catch (IOException e) { // expected } assertEquals(0, fs.getNumberOfOpenInputStreams()); assertEquals(0, fs.getNumberOfOpenOutputStreams()); assertEquals(0, fs.getTotalNumberOfOpenStreams()); }
@Override public FileSystem create(URI fsUri) throws IOException { FileSystem original = factory.create(fsUri); return new LimitedConnectionsFileSystem(original, settings.limitTotal, settings.limitOutput, settings.limitInput, settings.streamOpenTimeout, settings.streamInactivityTimeout); }
@Override public FileSystem create(URI fsUri) throws IOException { FileSystem original = factory.create(fsUri); return new LimitedConnectionsFileSystem(original, settings.limitTotal, settings.limitOutput, settings.limitInput, settings.streamOpenTimeout, settings.streamInactivityTimeout); }
private static FileSystem limitIfConfigured(HadoopFileSystem fs, String scheme, Configuration config) { final ConnectionLimitingSettings limitSettings = ConnectionLimitingSettings.fromConfig(config, scheme); // decorate only if any limit is configured if (limitSettings == null) { // no limit configured return fs; } else { return new LimitedConnectionsFileSystem( fs, limitSettings.limitTotal, limitSettings.limitOutput, limitSettings.limitInput, limitSettings.streamOpenTimeout, limitSettings.streamInactivityTimeout); } } }