/** * Delete a file or directory and its contents recursively. * Don't follow directories if they are symlinks. * * @param file Input file / dir to be deleted * @throws IOException if deletion is unsuccessful */ public static void deleteRecursively(File file) throws IOException { deleteRecursively(file, null); }
final byte[] key = JavaUtils.bufferToArray(entry.getKey()); final byte[] value = entry.getValue(); final BytesToBytesMap.Location loc =
/** * Convert a passed byte string (e.g. 50b, 100k, or 250m) to bytes for * internal use. * * If no suffix is provided, the passed number is assumed to be in bytes. */ public static long byteStringAsBytes(String str) { return byteStringAs(str, ByteUnit.BYTE); }
@Override public void receive( TransportClient client, ByteBuffer message, RpcResponseCallback callback) { assertEquals("Ping", JavaUtils.bytesToString(message)); callback.onSuccess(JavaUtils.stringToBytes("Pong")); }
private static void deleteRecursivelyUsingJavaIO(File file) throws IOException { if (file.isDirectory() && !isSymlink(file)) { IOException savedIOException = null; for (File child : listFilesSafely(file)) { try { deleteRecursively(child); } catch (IOException e) { // In case of multiple exceptions, only last one will be thrown savedIOException = e; } } if (savedIOException != null) { throw savedIOException; } } boolean deleted = file.delete(); // Delete can also fail if the file simply did not exist. if (!deleted && file.exists()) { throw new IOException("Failed to delete: " + file.getAbsolutePath()); } }
/** * Register an application with its secret specified as a byte buffer. */ public void registerApp(String appId, ByteBuffer shuffleSecret) { registerApp(appId, JavaUtils.bytesToString(shuffleSecret)); }
/** Close all connections in the connection pool, and shutdown the worker thread pool. */ @Override public void close() { // Go through all clients and close them if they are active. for (ClientPool clientPool : connectionPool.values()) { for (int i = 0; i < clientPool.clients.length; i++) { TransportClient client = clientPool.clients[i]; if (client != null) { clientPool.clients[i] = null; JavaUtils.closeQuietly(client); } } } connectionPool.clear(); if (workerGroup != null) { workerGroup.shutdownGracefully(); workerGroup = null; } } }
/** * Maximum number of bytes to be encrypted at a time when SASL encryption is enabled. */ public int maxSaslEncryptedBlockSize() { return Ints.checkedCast(JavaUtils.byteStringAsBytes( conf.get("spark.network.sasl.maxEncryptedBlockSize", "64k"))); }
/** * Delete a file or directory and its contents recursively. * Don't follow directories if they are symlinks. * * @param file Input file / dir to be deleted * @param filter A filename filter that make sure only files / dirs with the satisfied filenames * are deleted. * @throws IOException if deletion is unsuccessful */ public static void deleteRecursively(File file, FilenameFilter filter) throws IOException { if (file == null) { return; } // On Unix systems, use operating system command to run faster // If that does not work out, fallback to the Java IO way if (SystemUtils.IS_OS_UNIX && filter == null) { try { deleteRecursivelyUsingUnixNative(file); return; } catch (IOException e) { logger.warn("Attempt to delete using native Unix OS command failed for path = {}. " + "Falling back to Java IO way", file.getAbsolutePath(), e); } } deleteRecursivelyUsingJavaIO(file, filter); }
@Test public void testDataEncryptionIsActuallyEnabled() throws Exception { // This test sets up an encrypted connection but then, using a client bootstrap, removes // the encryption handler from the client side. This should cause the server to not be // able to understand RPCs sent to it and thus close the connection. SaslTestCtx ctx = null; try { ctx = new SaslTestCtx(mock(RpcHandler.class), true, true); ctx.client.sendRpcSync(JavaUtils.stringToBytes("Ping"), TimeUnit.SECONDS.toMillis(10)); fail("Should have failed to send RPC to server."); } catch (Exception e) { assertFalse(e.getCause() instanceof TimeoutException); } finally { if (ctx != null) { ctx.close(); } } }
@Override public void receive( TransportClient client, ByteBuffer message, RpcResponseCallback callback) { assertEquals("Ping", JavaUtils.bytesToString(message)); callback.onSuccess(JavaUtils.stringToBytes("Pong")); }
private static void deleteRecursivelyUsingJavaIO(File file) throws IOException { if (file.isDirectory() && !isSymlink(file)) { IOException savedIOException = null; for (File child : listFilesSafely(file)) { try { deleteRecursively(child); } catch (IOException e) { // In case of multiple exceptions, only last one will be thrown savedIOException = e; } } if (savedIOException != null) { throw savedIOException; } } boolean deleted = file.delete(); // Delete can also fail if the file simply did not exist. if (!deleted && file.exists()) { throw new IOException("Failed to delete: " + file.getAbsolutePath()); } }
/** * Register an application with its secret specified as a byte buffer. */ public void registerApp(String appId, ByteBuffer shuffleSecret) { registerApp(appId, JavaUtils.bytesToString(shuffleSecret)); }
/** Close all connections in the connection pool, and shutdown the worker thread pool. */ @Override public void close() { // Go through all clients and close them if they are active. for (ClientPool clientPool : connectionPool.values()) { for (int i = 0; i < clientPool.clients.length; i++) { TransportClient client = clientPool.clients[i]; if (client != null) { clientPool.clients[i] = null; JavaUtils.closeQuietly(client); } } } connectionPool.clear(); if (workerGroup != null) { workerGroup.shutdownGracefully(); workerGroup = null; } } }
/** * Minimum size of a block that we should start using memory map rather than reading in through * normal IO operations. This prevents Spark from memory mapping very small blocks. In general, * memory mapping has high overhead for blocks close to or below the page size of the OS. */ public int memoryMapBytes() { return Ints.checkedCast(JavaUtils.byteStringAsBytes( conf.get("spark.storage.memoryMapThreshold", "2m"))); }
/** * Delete a file or directory and its contents recursively. * Don't follow directories if they are symlinks. * * @param file Input file / dir to be deleted * @throws IOException if deletion is unsuccessful */ public static void deleteRecursively(File file) throws IOException { if (file == null) { return; } // On Unix systems, use operating system command to run faster // If that does not work out, fallback to the Java IO way if (SystemUtils.IS_OS_UNIX) { try { deleteRecursivelyUsingUnixNative(file); return; } catch (IOException e) { logger.warn("Attempt to delete using native Unix OS command failed for path = {}. " + "Falling back to Java IO way", file.getAbsolutePath(), e); } } deleteRecursivelyUsingJavaIO(file); }
@Test public void sendOneWayMessage() throws Exception { final String message = "no reply"; TransportClient client = clientFactory.createClient(TestUtils.getLocalHost(), server.getPort()); try { client.send(JavaUtils.stringToBytes(message)); assertEquals(0, client.getHandler().numOutstandingRequests()); // Make sure the message arrives. long deadline = System.nanoTime() + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS); while (System.nanoTime() < deadline && oneWayMsgs.size() == 0) { TimeUnit.MILLISECONDS.sleep(10); } assertEquals(1, oneWayMsgs.size()); assertEquals(message, oneWayMsgs.get(0)); } finally { client.close(); } }
@Override public void receive( TransportClient client, ByteBuffer message, RpcResponseCallback callback) { String msg = JavaUtils.bytesToString(message); String[] parts = msg.split("/"); if (parts[0].equals("hello")) { callback.onSuccess(JavaUtils.stringToBytes("Hello, " + parts[1] + "!")); } else if (parts[0].equals("return error")) { callback.onFailure(new RuntimeException("Returned: " + parts[1])); } else if (parts[0].equals("throw error")) { throw new RuntimeException("Thrown: " + parts[1]); } }
private static void deleteRecursivelyUsingJavaIO( File file, FilenameFilter filter) throws IOException { if (file.isDirectory() && !isSymlink(file)) { IOException savedIOException = null; for (File child : listFilesSafely(file, filter)) { try { deleteRecursively(child, filter); } catch (IOException e) { // In case of multiple exceptions, only last one will be thrown savedIOException = e; } } if (savedIOException != null) { throw savedIOException; } } // Delete file only when it's a normal file or an empty directory. if (file.isFile() || (file.isDirectory() && listFilesSafely(file, null).length == 0)) { boolean deleted = file.delete(); // Delete can also fail if the file simply did not exist. if (!deleted && file.exists()) { throw new IOException("Failed to delete: " + file.getAbsolutePath()); } } }
/** * Delete a file or directory and its contents recursively. * Don't follow directories if they are symlinks. * * @param file Input file / dir to be deleted * @throws IOException if deletion is unsuccessful */ public static void deleteRecursively(File file) throws IOException { deleteRecursively(file, null); }