/** * Create a new PDFS protocol instance. * * @param endpoint the local endpoint * @param config the Sabot configuration * @param allocator the memory allocator to use. The caller is in charge of closing it * @return the protocol * @throws IOException */ public static PDFSProtocol newInstance(NodeEndpoint endpoint, SabotConfig config, BufferAllocator allocator, boolean allowLocalHandling) throws IOException { // we'll grab a raw local file system so append is supported (rather than // the checksum local file system). Configuration conf = new Configuration(); return new PDFSProtocol(endpoint, config, allocator, PseudoDistributedFileSystem.newLocalFileSystem(conf, allowLocalHandling), allowLocalHandling, Ticker.systemTicker()); }
return handle(connection, DFS.GetFileStatusRequest.parseFrom(pBody)); return handle(connection, DFS.GetFileDataRequest.parseFrom(pBody)); return handle(connection, DFS.ListStatusRequest.parseFrom(pBody)); return handle(connection, DFS.MkdirsRequest.parseFrom(pBody)); return handle(connection, DFS.RenameRequest.parseFrom(pBody)); return handle(connection, DFS.DeleteRequest.parseFrom(pBody)); return handle(connection, DFS.WriteDataRequest.parseFrom(pBody), dBody);
@Override public void start() throws Exception { FabricService fabricService = this.fabricService.get(); pool = new CloseableThreadPool("pdfs"); FabricRunnerFactory factory = fabricService.registerProtocol(PDFSProtocol.newInstance(identityProvider.get(), this.config, allocator, allowLocalAccess)); final PDFSConfig config = new PDFSConfig(pool, factory, allocator, nodeProvider, identityProvider.get(), allowLocalAccess); PseudoDistributedFileSystem.configure(config); // A hack until DX-4639 is addressed. createFileSystem(); }
private Response writeData(Path path, ByteBuf buf, boolean create) throws IOException { try(FSDataOutputStream output = create ? localFS.create(path) : localFS.append(path)) { byte[] bytes = new byte[buf.readableBytes()]; buf.readBytes(bytes); output.write(bytes); } final FileStatus fs = localFS.getFileStatus(path); final WriteDataResponse response = WriteDataResponse.newBuilder().setUpdateTime(fs.getModificationTime()).build(); return reply(DFS.RpcType.WRITE_DATA_RESPONSE, response); }
@Test public void testStream() throws IOException { TestRemoteIterator statuses = newRemoteIterator( new FileStatus(1337, false, 1, 4096, 1, 2, FsPermission.getFileDefault(), "testowner", "testgroup", new Path(TEST_PATH, "bar")), new FileStatus(0, true, 0, 0, 3, 4, FsPermission.getDirDefault(), "testowner", "testgroup", new Path(TEST_PATH, "baz")) ); final ListStatusContinuationHandle handle; { DFS.ListStatusResponse response = getResponse(TEST_PATH_STRING, statuses, 1, null); assertEquals(1, response.getStatusesList().size()); assertEquals(TEST_PATH_STRING + "/bar", response.getStatusesList().get(0).getPath()); assertTrue(response.hasHandle()); handle = response.getHandle(); assertTrue(getPDFSProtocol().isIteratorOpen(handle)); assertFalse(statuses.isClosed()); } { DFS.ListStatusResponse response = getResponse(TEST_PATH_STRING, null, 1, handle); assertEquals(1, response.getStatusesList().size()); assertEquals(TEST_PATH_STRING + "/baz", response.getStatusesList().get(0).getPath()); assertFalse(response.hasHandle()); assertFalse(getPDFSProtocol().isIteratorOpen(handle)); assertTrue(statuses.isClosed()); } }
private Response handle(PhysicalConnection connection, DFS.WriteDataRequest request, ByteBuf buf) throws IOException { final Path path = new Path(request.getPath()); if(request.getLastOffset() == 0){ // initial creation and write. return writeData(path, buf, true); } // append, first check last update time and offset. (concurrency danger between check and write but doesn't // seem important in this usecase (home file uploads).) FileStatus fs = localFS.getFileStatus(path); if(fs.getModificationTime() != request.getLastUpdate()){ throw new IOException(String.format("Unexpected last modification time. Expected time: %d, Actual time: %d.", request.getLastUpdate(), fs.getModificationTime())); } if(fs.getLen() != request.getLastOffset()) { throw new IOException(String.format("Unexpected last offset. Remote offset: %d, Actual offset: %d.", request.getLastOffset(), fs.getLen())); } return writeData(path, buf, false); }
private Response handle(PhysicalConnection connection, DFS.DeleteRequest request) throws IOException { Path path = new Path(request.getPath()); Boolean recursive = request.getRecursive(); boolean result = localFS.delete(path, recursive != null ? recursive.booleanValue() : false); return reply(DFS.RpcType.DELETE_RESPONSE, DFS.DeleteResponse.newBuilder().setValue(result).build()); }
@Test public void testStreamOptionalPath() throws IOException { TestRemoteIterator statuses = newRemoteIterator( new FileStatus(1337, false, 1, 4096, 1, 2, FsPermission.getFileDefault(), "testowner", "testgroup", new Path(TEST_PATH, "bar")), new FileStatus(0, true, 0, 0, 3, 4, FsPermission.getDirDefault(), "testowner", "testgroup", new Path(TEST_PATH, "baz")) ); final ListStatusContinuationHandle handle; { DFS.ListStatusResponse response = getResponse(TEST_PATH_STRING, statuses, 1, null); assertEquals(1, response.getStatusesList().size()); assertEquals(TEST_PATH_STRING + "/bar", response.getStatusesList().get(0).getPath()); assertTrue(response.hasHandle()); handle = response.getHandle(); assertTrue(getPDFSProtocol().isIteratorOpen(handle)); assertFalse(statuses.isClosed()); } { DFS.ListStatusResponse response = getResponse(null, null, 1, handle); assertEquals(1, response.getStatusesList().size()); assertEquals(TEST_PATH_STRING + "/baz", response.getStatusesList().get(0).getPath()); assertFalse(response.hasHandle()); assertFalse(getPDFSProtocol().isIteratorOpen(handle)); assertTrue(statuses.isClosed()); } }
/** * Get the response produced by the handler for a given filesystem response. * * @return * @throws IOException * @throws UserException */ private DFS.GetFileStatusResponse getResponse(final Object o) throws IOException, UserException { if (o instanceof Throwable) { doThrow((Throwable) o).when(getFileSystem()).getFileStatus(TEST_PATH); } else { doReturn(o).when(getFileSystem()).getFileStatus(TEST_PATH); } Response response = getPDFSProtocol().handle(getConnection(), DFS.RpcType.GET_FILE_STATUS_REQUEST_VALUE, DFS.GetFileStatusRequest.newBuilder().setPath(TEST_PATH_STRING).build().toByteString(), null); assertEquals(DFS.RpcType.GET_FILE_STATUS_RESPONSE, response.rpcType); assertArrayEquals(new ByteBuf[] {}, response.dBodies); return (DFS.GetFileStatusResponse) response.pBody; }
@Before public void setUpPDFSService() throws IOException { NodeEndpoint endpoint = NodeEndpoint.newBuilder() .setAddress(fabric.getAddress()).setFabricPort(fabric.getPort()) .build(); PDFSProtocol pdfsProtocol = PDFSProtocol.newInstance(endpoint, DremioTest.DEFAULT_SABOT_CONFIG, allocator, true); FabricRunnerFactory factory = fabric.registerProtocol(pdfsProtocol); sabotFS = new RemoteNodeFileSystem(factory.getCommandRunner(fabric.getAddress(), fabric.getPort()), allocator); sabotFS.initialize(URI.create(format("sabot://%s:%d", fabric.getAddress(), fabric.getPort())), new Configuration(false)); }
@Before public void setUp() throws IOException { this.ticker = new TestTicker(); pdfsProtocol = new PDFSProtocol(LOCAL_ENDPOINT, DremioTest.DEFAULT_SABOT_CONFIG, this.allocator, fileSystem, true, ticker); }
private Response handle(PhysicalConnection connection, DFS.RenameRequest request) throws IOException { Path oldPath = new Path(request.getOldpath()); Path newPath = new Path(request.getNewpath()); boolean result = localFS.rename(oldPath, newPath); return reply(DFS.RpcType.RENAME_RESPONSE, DFS.RenameResponse.newBuilder().setValue(result).build()); }
@Test public void testStreamWithCacheExpiration() throws IOException { TestRemoteIterator statuses = newRemoteIterator( new FileStatus(1337, false, 1, 4096, 1, 2, FsPermission.getFileDefault(), "testowner", "testgroup", new Path(TEST_PATH, "bar")), new FileStatus(0, true, 0, 0, 3, 4, FsPermission.getDirDefault(), "testowner", "testgroup", new Path(TEST_PATH, "baz")) ); final ListStatusContinuationHandle handle; { DFS.ListStatusResponse response = getResponse(TEST_PATH_STRING, statuses, 1, null); assertEquals(1, response.getStatusesList().size()); assertEquals(TEST_PATH_STRING + "/bar", response.getStatusesList().get(0).getPath()); assertTrue(response.hasHandle()); handle = response.getHandle(); assertTrue(getPDFSProtocol().isIteratorOpen(handle)); assertFalse(statuses.isClosed()); } // Moving ticker so that open iterators expire getTicker().advance(5, TimeUnit.MINUTES); try { getResponse(TEST_PATH_STRING, null, 1, handle); fail(); } catch (UserException e) { assertEquals(ErrorType.IO_EXCEPTION, e.getErrorType()); assertSame(IOException.class, e.getCause().getClass()); } }
/** * Get the response produced by the handler for a given filesystem response. * * @return * @throws IOException * @throws UserException */ private Response getResponse(Long start, Integer length, final Object o) throws IOException, UserException { if (o instanceof Throwable) { doThrow((Throwable) o).when(getFileSystem()).open(TEST_PATH); } else { doReturn(o).when(getFileSystem()).open(TEST_PATH); } Response response = getPDFSProtocol().handle(getConnection(), DFS.RpcType.GET_FILE_DATA_REQUEST_VALUE, DFS.GetFileDataRequest.newBuilder().setPath(TEST_PATH_STRING).setStart(start).setLength(length).build().toByteString(), null); assertEquals(DFS.RpcType.GET_FILE_DATA_RESPONSE, response.rpcType); //assertArrayEquals(new ByteBuf[] {}, response.dBodies); return response; }
private Response handle(PhysicalConnection connection, DFS.GetFileStatusRequest request) throws IOException { Path path = new Path(request.getPath()); FileStatus status = localFS.getFileStatus(path); DFS.GetFileStatusResponse response = DFS.GetFileStatusResponse.newBuilder() .setStatus(RemoteNodeFileSystem.toProtoFileStatus(status)) .build(); return reply(DFS.RpcType.GET_FILE_STATUS_RESPONSE, response); }
/** * Get the response produced by the handler for a given filesystem response. * * @return * @throws IOException * @throws UserException */ private DFS.DeleteResponse getResponse(final boolean recursive, final Object o) throws IOException, UserException { if (o instanceof Throwable) { doThrow((Throwable) o).when(getFileSystem()).delete(TEST_PATH, recursive); } else { doReturn(o).when(getFileSystem()).delete(TEST_PATH, recursive); } Response response = getPDFSProtocol().handle(getConnection(), DFS.RpcType.DELETE_REQUEST_VALUE, DFS.DeleteRequest.newBuilder().setPath(TEST_PATH_STRING).setRecursive(recursive).build().toByteString(), null); assertEquals(DFS.RpcType.DELETE_RESPONSE, response.rpcType); assertArrayEquals(new ByteBuf[] {}, response.dBodies); return (DFS.DeleteResponse) response.pBody; }
private Response handle(PhysicalConnection connection, DFS.GetFileDataRequest request) throws IOException { Path path = new Path(request.getPath()); try(FSDataInputStream fdis = localFS.open(path)) { byte[] buf = new byte[request.getLength()]; fdis.seek(request.getStart()); int read = fdis.read(buf); DFS.GetFileDataResponse response = DFS.GetFileDataResponse.newBuilder().setRead(read).build(); ByteBuf[] bodies = (read != -1) ? new ByteBuf[] { Unpooled.wrappedBuffer(buf, 0, read) } : NO_BUFS; return reply(DFS.RpcType.GET_FILE_DATA_RESPONSE, response, bodies); } }
/** * Get the response produced by the handler for a given filesystem response. * * @return * @throws IOException * @throws UserException */ private DFS.RenameResponse getResponse(final Object o) throws IOException, UserException { if (o instanceof Throwable) { doThrow((Throwable) o).when(getFileSystem()).rename(TEST_PATH, TEST_PATH_2); } else { doReturn(o).when(getFileSystem()).rename(TEST_PATH, TEST_PATH_2); } Response response = getPDFSProtocol().handle(getConnection(), DFS.RpcType.RENAME_REQUEST_VALUE, DFS.RenameRequest.newBuilder().setOldpath(TEST_PATH_STRING).setNewpath(TEST_PATH_2_STRING).build().toByteString(), null); assertEquals(DFS.RpcType.RENAME_RESPONSE, response.rpcType); assertArrayEquals(new ByteBuf[] {}, response.dBodies); return (DFS.RenameResponse) response.pBody; }
private Response handle(PhysicalConnection connection, DFS.MkdirsRequest request) throws IOException { Path path = new Path(request.getPath()); FsPermission permission = RemoteNodeFileSystem.toFsPermission(request.getPermission()); boolean result = localFS.mkdirs(path, permission); return reply(DFS.RpcType.MKDIRS_RESPONSE, DFS.MkdirsResponse.newBuilder().setValue(result).build()); }
/** * Get the response produced by the handler for a given filesystem response. * * @return * @throws IOException * @throws UserException */ private DFS.MkdirsResponse getResponse(final FsPermission permission, final Object o) throws IOException, UserException { if (o instanceof Throwable) { doThrow((Throwable) o).when(getFileSystem()).mkdirs(TEST_PATH, permission); } else { doReturn(o).when(getFileSystem()).mkdirs(TEST_PATH, permission); } Response response = getPDFSProtocol().handle(getConnection(), DFS.RpcType.MKDIRS_REQUEST_VALUE, DFS.MkdirsRequest.newBuilder().setPath(TEST_PATH_STRING).setPermission(permission.toExtendedShort()).build().toByteString(), null); assertEquals(DFS.RpcType.MKDIRS_RESPONSE, response.rpcType); assertArrayEquals(new ByteBuf[] {}, response.dBodies); return (DFS.MkdirsResponse) response.pBody; }