/** * validate if a file on HDFS can be open * * @throws IOException the file can't be found or opened * @throws URISyntaxException */ public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException { try(InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(new URI(pathStr)))) { if (inputStream == null) { throw new IOException("HDFS file " + pathStr + " can't be opened"); } } }
/** * validate if a file on HDFS can be open * * @throws IOException the file can't be found or opened * @throws URISyntaxException */ public static void validateHDFSFile(SliderFileSystem sliderFileSystem, String pathStr) throws IOException, URISyntaxException { try(InputStream inputStream = sliderFileSystem.getFileSystem().open(new Path(new URI(pathStr)))) { if (inputStream == null) { throw new IOException("HDFS file " + pathStr + " can't be opened"); } } }
private void uploadServerCertForLocalization(String clustername, SliderFileSystem fs) throws IOException { Path certsDir = fs.buildClusterSecurityDirPath(clustername); if (!fs.getFileSystem().exists(certsDir)) { fs.getFileSystem().mkdirs(certsDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); } Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME); if (!fs.getFileSystem().exists(destPath)) { fs.getFileSystem().copyFromLocalFile( new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()), destPath); log.info("Uploaded server cert to localization path {}", destPath); } fs.getFileSystem().setPermission(destPath, new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); }
private void uploadServerCertForLocalization(String clustername, SliderFileSystem fs) throws IOException { Path certsDir = fs.buildClusterSecurityDirPath(clustername); if (!fs.getFileSystem().exists(certsDir)) { fs.getFileSystem().mkdirs(certsDir, new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); } Path destPath = new Path(certsDir, SliderKeys.CRT_FILE_NAME); if (!fs.getFileSystem().exists(destPath)) { fs.getFileSystem().copyFromLocalFile( new Path(CertificateManager.getServerCertficateFilePath().getAbsolutePath()), destPath); log.info("Uploaded server cert to localization path {}", destPath); } fs.getFileSystem().setPermission(destPath, new FsPermission(FsAction.READ, FsAction.NONE, FsAction.NONE)); }
static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem, String appDef, String configFileName) throws IOException { // this is the path inside the zip file String fileToRead = "configuration/" + configFileName; log.info("Reading default config file {} at {}", fileToRead, appDef); InputStream configStream = SliderUtils.getApplicationResourceInputStream( fileSystem.getFileSystem(), new Path(appDef), fileToRead); if (configStream == null) { log.error("{} is unavailable at {}.", fileToRead, appDef); throw new IOException("Expected config file " + fileToRead + " is not available."); } return new DefaultConfigParser().parse(configStream); }
private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws YarnException, IOException { requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, actionPackageArgs.version); FileSystem fs = sliderFileSystem.getFileSystem(); require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri()); log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath); if(fs.delete(pkgPath, true)) { log.info("Deleted package {} ", actionPackageArgs.name); return EXIT_SUCCESS; } else { log.warn("Package deletion failed."); return EXIT_NOT_FOUND; } }
private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws YarnException, IOException { requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name); Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name, actionPackageArgs.version); FileSystem fs = sliderFileSystem.getFileSystem(); require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri()); log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath); if(fs.delete(pkgPath, true)) { log.info("Deleted package {} " + actionPackageArgs.name); return EXIT_SUCCESS; } else { log.warn("Package deletion failed."); return EXIT_NOT_FOUND; } }
static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem, String appDef, String configFileName) throws IOException { // this is the path inside the zip file String fileToRead = "configuration/" + configFileName; log.info("Reading default config file {} at {}", fileToRead, appDef); InputStream configStream = SliderUtils.getApplicationResourceInputStream( fileSystem.getFileSystem(), new Path(appDef), fileToRead); if (configStream == null) { log.error("{} is unavailable at {}.", fileToRead, appDef); throw new IOException("Expected config file " + fileToRead + " is not available."); } return new DefaultConfigParser().parse(configStream); }
private int actionDeleteResource(ActionResourceArgs resourceInfo) throws BadCommandArgumentsException, IOException { if (StringUtils.isEmpty(resourceInfo.resource)) { throw new BadCommandArgumentsException("A file name is required."); } Path fileInFs; if (resourceInfo.folder == null) { fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource); } else { fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder, resourceInfo.resource); } log.info("Deleting resource {}", fileInFs); FileSystem sfs = sliderFileSystem.getFileSystem(); require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri()); sfs.delete(fileInFs, true); return EXIT_SUCCESS; }
private int actionDeleteResource(ActionResourceArgs resourceInfo) throws BadCommandArgumentsException, IOException { if (StringUtils.isEmpty(resourceInfo.resource)) { throw new BadCommandArgumentsException("A file name is required."); } Path fileInFs; if (resourceInfo.folder == null) { fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.resource); } else { fileInFs = sliderFileSystem.buildResourcePath(resourceInfo.folder, resourceInfo.resource); } log.info("Deleting resource {}", fileInFs); FileSystem sfs = sliderFileSystem.getFileSystem(); require(sfs.exists(fileInFs), "No resource to delete found at %s", fileInFs.toUri()); sfs.delete(fileInFs, true); return EXIT_SUCCESS; }
private int actionListResource(ActionResourceArgs resourceInfo) throws IOException { String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY; Path path = sliderFileSystem.buildResourcePath(folder); RemoteIterator<LocatedFileStatus> files = sliderFileSystem.getFileSystem().listFiles(path, true); log.info("Resources:"); while (files.hasNext()) { log.info("\t" + files.next().getPath().toString()); } return EXIT_SUCCESS; }
private int actionListKeytab(ActionKeytabArgs keytabInfo) throws IOException { String folder = keytabInfo.folder != null ? keytabInfo.folder : StringUtils.EMPTY; Path keytabPath = sliderFileSystem.buildKeytabInstallationDirPath(folder); RemoteIterator<LocatedFileStatus> files = sliderFileSystem.getFileSystem().listFiles(keytabPath, true); log.info("Keytabs:"); while (files.hasNext()) { log.info("\t" + files.next().getPath().toString()); } return EXIT_SUCCESS; }
private int actionListKeytab(ActionKeytabArgs keytabInfo) throws IOException { String folder = keytabInfo.folder != null ? keytabInfo.folder : StringUtils.EMPTY; Path keytabPath = sliderFileSystem.buildKeytabInstallationDirPath(folder); RemoteIterator<LocatedFileStatus> files = sliderFileSystem.getFileSystem().listFiles(keytabPath, true); log.info("Keytabs:"); while (files.hasNext()) { log.info("\t" + files.next().getPath().toString()); } return EXIT_SUCCESS; }
private int actionListResource(ActionResourceArgs resourceInfo) throws IOException { String folder = resourceInfo.folder != null ? resourceInfo.folder : StringUtils.EMPTY; Path path = sliderFileSystem.buildResourcePath(folder); RemoteIterator<LocatedFileStatus> files = sliderFileSystem.getFileSystem().listFiles(path, true); log.info("Resources:"); while (files.hasNext()) { log.info("\t" + files.next().getPath().toString()); } return EXIT_SUCCESS; }
/** * Save/list tokens. This is for testing oozie integration * @param args commands * @return status */ private int actionTokens(ActionTokensArgs args) throws IOException, YarnException { return new TokensOperation().actionTokens(args, sliderFileSystem.getFileSystem(), getConfig(), yarnClient); }
/** * Save/list tokens. This is for testing oozie integration * @param args commands * @return status */ private int actionTokens(ActionTokensArgs args) throws IOException, YarnException { return new TokensOperation().actionTokens(args, sliderFileSystem.getFileSystem(), getConfig(), yarnClient); }
/** * Build the credentials needed for containers. This will include * getting new delegation tokens for HDFS if the AM is running * with a keytab. * @return a buffer of credentials * @throws IOException */ private Credentials buildContainerCredentials() throws IOException { Credentials credentials = new Credentials(containerCredentials); if (securityConfiguration.isKeytabProvided()) { CredentialUtils.addSelfRenewableFSDelegationTokens( getClusterFS().getFileSystem(), credentials); } return credentials; }
/** * Build the credentials needed for containers. This will include * getting new delegation tokens for HDFS if the AM is running * with a keytab. * @return a buffer of credentials * @throws IOException */ private Credentials buildContainerCredentials() throws IOException { Credentials credentials = new Credentials(containerCredentials); if (securityConfiguration.isKeytabProvided()) { CredentialUtils.addSelfRenewableFSDelegationTokens( getClusterFS().getFileSystem(), credentials); } return credentials; }
@Test public void testGetMetaInfoStreamFromZip() throws Exception { String zipFileName = TestUtility.createAppPackage( folder, "testpkg", "test.zip", "target/test-classes/org/apache/slider/common/tools/test"); Configuration configuration = new Configuration(); FileSystem fs = FileSystem.getLocal(configuration); log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); InputStream stream = SliderUtils.getApplicationResourceInputStream( sliderFileSystem.getFileSystem(), new Path(zipFileName), "metainfo.xml"); Assert.assertTrue(stream != null); Assert.assertTrue(stream.available() > 0); }
@Test public void testGetMetaInfoStreamFromZip() throws Exception { String zipFileName = TestUtility.createAppPackage( folder, "testpkg", "test.zip", "target/test-classes/org/apache/slider/common/tools/test"); Configuration configuration = new Configuration(); FileSystem fs = FileSystem.getLocal(configuration); log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); InputStream stream = SliderUtils.getApplicationResourceInputStream( sliderFileSystem.getFileSystem(), new Path(zipFileName), "metainfo.xml"); Assert.assertTrue(stream != null); Assert.assertTrue(stream.available() > 0); }