/** * Executes a sequence of commands on the remote host and block until execution completed. * * @param commands the commands to execute * @return the output to stdout by the commands * @throws IOException if failed to execute command or command exit with non-zero values. */ default String executeAndWait(String... commands) throws IOException { return executeAndWait(Arrays.asList(commands)); }
@Override public void copy(InputStream input, String targetPath, String targetName, long size, int permission, @Nullable Long lastAccessTime, @Nullable Long lastModifiedTime) throws IOException { getDelegate().copy(input, targetPath, targetName, size, permission, lastAccessTime, lastModifiedTime); }
@Override public PortForwarding createLocalPortForward(String targetHost, int targetPort, int originatePort, PortForwarding.DataConsumer dataConsumer) throws IOException { return getDelegate().createLocalPortForward(targetHost, targetPort, originatePort, dataConsumer); }
session.executeAndWait("mkdir -p " + localizedDir); localizedFile = localizedDir + "/" + fileName; try (InputStream inputStream = openURI(uri)) { LOG.debug("Upload file {} to {}@{}:{}", uri, session.getUsername(), session.getAddress(), localizedFile); session.copy(inputStream, localizedDir, fileName, localFile.getSize(), 0644, localFile.getLastModified(), localFile.getLastModified()); String expandedDir = targetPath + "/" + localFile.getName(); LOG.debug("Expanding archive {} on host {} to {}", localizedFile, session.getAddress().getHostName(), expandedDir); session.executeAndWait( "mkdir -p " + expandedDir, "cd " + expandedDir, } else { LOG.debug("Create hardlink {} on host {} to {}/{}", localizedFile, session.getAddress().getHostName(), targetPath, localFile.getName()); session.executeAndWait(String.format("ln %s %s/%s", localizedFile, targetPath, localFile.getName()));
String targetPath = session.executeAndWait("mkdir -p ./" + programRunId.getRun(), "echo `pwd`/" + programRunId.getRun()).trim(); session.executeAndWait(String.format("bash %s/%s/%s %s/%s/%s > %s/%s", targetPath, Constants.Files.RUNTIME_CONFIG_JAR, SETUP_SPARK_SH, targetPath, Constants.Files.RUNTIME_CONFIG_JAR, SETUP_SPARK_PY, runnableName, memory).getBytes(StandardCharsets.UTF_8); session.copy(new ByteArrayInputStream(scriptContent), targetPath, "launcher.sh", scriptContent.length, 0755, null, null); LOG.info("Starting runnable {} with SSH on {}", runnableName, session.getAddress().getHostName()); session.executeAndWait(targetPath + "/launcher.sh");
/** * Executes a sequence of commands on the remote host. * * @param commands the commands to execute * @return the command result * @throws IOException if failed to execute command remotely */ default SSHProcess execute(String... commands) throws IOException { return execute(Arrays.asList(commands)); }
/** * Close this manager by close all {@link SSHSession}s that are managed by this class. */ @Override public void close() { for (SSHInfo info : sshInfos.values()) { CloseDisabledSSHSession session = info.getSession(); if (session != null) { session.getDelegate().close(); } } sshInfos.clear(); }
@Override public InetSocketAddress getAddress() { return getDelegate().getAddress(); }
session.executeAndWait("mkdir -p " + localizedDir); localizedFile = localizedDir + "/" + fileName; try (InputStream inputStream = openURI(uri)) { LOG.debug("Upload file {} to {}@{}:{}", uri, session.getUsername(), session.getAddress(), localizedFile); session.copy(inputStream, localizedDir, fileName, localFile.getSize(), 0644, localFile.getLastModified(), localFile.getLastModified()); String expandedDir = targetPath + "/" + localFile.getName(); LOG.debug("Expanding archive {} on host {} to {}", localizedFile, session.getAddress().getHostName(), expandedDir); session.executeAndWait( "mkdir -p " + expandedDir, "cd " + expandedDir, } else { LOG.debug("Create hardlink {} on host {} to {}/{}", localizedFile, session.getAddress().getHostName(), targetPath, localFile.getName()); session.executeAndWait(String.format("ln %s %s/%s", localizedFile, targetPath, localFile.getName()));
String targetPath = session.executeAndWait("mkdir -p ./" + programRunId.getRun(), "echo `pwd`/" + programRunId.getRun()).trim(); session.executeAndWait(String.format("bash %s/%s/%s %s/%s/%s > %s/%s", targetPath, Constants.Files.RUNTIME_CONFIG_JAR, SETUP_SPARK_SH, targetPath, Constants.Files.RUNTIME_CONFIG_JAR, SETUP_SPARK_PY, runnableName, memory).getBytes(StandardCharsets.UTF_8); session.copy(new ByteArrayInputStream(scriptContent), targetPath, "launcher.sh", scriptContent.length, 0755, null, null); LOG.info("Starting runnable {} with SSH on {}", runnableName, session.getAddress().getHostName()); session.executeAndWait(targetPath + "/launcher.sh");
@Override public SSHProcess execute(List<String> commands) throws IOException { return getDelegate().execute(commands); }
/** * Close this manager by close all {@link SSHSession}s that are managed by this class. */ @Override public void close() { for (SSHInfo info : sshInfos.values()) { CloseDisabledSSHSession session = info.getSession(); if (session != null) { session.getDelegate().close(); } } sshInfos.clear(); }
@Override public InetSocketAddress getAddress() { return getDelegate().getAddress(); }
@Override public String executeAndWait(List<String> commands) throws IOException { return getDelegate().executeAndWait(commands); }
@Override public SSHProcess execute(List<String> commands) throws IOException { return getDelegate().execute(commands); }
@Override public void copy(InputStream input, String targetPath, String targetName, long size, int permission, @Nullable Long lastAccessTime, @Nullable Long lastModifiedTime) throws IOException { getDelegate().copy(input, targetPath, targetName, size, permission, lastAccessTime, lastModifiedTime); }
@Override public PortForwarding createLocalPortForward(String targetHost, int targetPort, int originatePort, PortForwarding.DataConsumer dataConsumer) throws IOException { return getDelegate().createLocalPortForward(targetHost, targetPort, originatePort, dataConsumer); }
/** * Removes a {@link SSHConfig} from this manager such that {@link SSHSession} cannot be acquired from the * {@link #getSession(InetSocketAddress)} method that goes to the same host. * This method will also InetSocketAddress the active {@link SSHSession} managed by this manager that is * associated with the given {@link SSHConfig}. * * @param serverAddr the {@link InetSocketAddress} of where the runtime monitor server is running */ void removeSSHConfig(InetSocketAddress serverAddr) { SSHInfo info = sshInfos.remove(serverAddr); CloseDisabledSSHSession session = info.getSession(); if (session != null) { session.getDelegate().close(); } }
@Override public String executeAndWait(List<String> commands) throws IOException { return getDelegate().executeAndWait(commands); }
@Override public void kill() throws Exception { // SSH and kill the process try (SSHSession session = new DefaultSSHSession(sshConfig)) { SSHProcess process = session.execute("pkill -9 -f -- -Dcdap.runid=" + programRunId.getRun()); // Reading will be blocked until the process finished ByteStreams.toByteArray(process.getInputStream()); String err = CharStreams.toString(new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8)); int exitCode = process.waitFor(); // If the exit code is 1, it means there is no such process, which is fine from the killing perspective if (exitCode == 0 || exitCode == 1) { return; } throw new IllegalStateException("Failed to kill remote process for program run " + programRunId + " due to error " + err); } } }