private Client(String[] args) throws IOException, ParseException, ClassNotFoundException { this.conf = new XLearningConfiguration(); this.dfs = FileSystem.get(conf); this.clientArguments = new ClientArguments(args); this.isRunning = new AtomicBoolean(false); this.appFilesRemotePath = new StringBuffer(1000); this.appLibJarsRemotePath = new StringBuffer(1000); this.inputPaths = new ConcurrentHashMap<>(); this.outputPaths = new ConcurrentHashMap<>(); JOB_FILE_PERMISSION = FsPermission.createImmutable((short) 0644); this.appMasterUserEnv = new HashMap<>(); this.containerUserEnv = new HashMap<>(); }
new OutputStreamWriter(fs.create(tmpPath, true), StandardCharsets.UTF_8))) { bw.write(json); fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644)); } catch (IOException e) { LOGGER.error("Unable to write to temp file " + tmpPath, e); fs.setPermission(path, FsPermission.createImmutable((short) 0644)); } catch (IOException e) { LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e);
private FileStatus createStatus(MockFile file) { if (fileStatusMap.containsKey(file)) { return fileStatusMap.get(file); } FileStatus fileStatus = new FileStatus(file.length, false, 1, file.blockSize, 0, 0, FsPermission.createImmutable((short) 644), "owen", "group", file.path); fileStatusMap.put(file, fileStatus); return fileStatus; }
@Nullable @Override public Object call() throws Exception { fs.setPermission(path, FsPermission.createImmutable((short)777)); return null; } }, IOException.class, "File system is stopped.");
private FileStatus createStatus(MockFile file) { if (fileStatusMap.containsKey(file)) { return fileStatusMap.get(file); } FileStatus fileStatus = new FileStatus(file.length, false, 1, file.blockSize, 0, 0, FsPermission.createImmutable((short) 644), "owen", "group", file.path); fileStatusMap.put(file, fileStatus); return fileStatus; }
private FileStatus createDirectory(Path dir) { return new FileStatus(0, true, 0, 0, 0, 0, FsPermission.createImmutable((short) 755), "owen", "group", dir); }
private FileStatus createDirectory(Path dir) { return new FileStatus(0, true, 0, 0, 0, 0, FsPermission.createImmutable((short) 755), "owen", "group", dir); }
private void testFSPerms() throws IOException { Path rootDir = FSUtils.getRootDir(conf); LOG.info(""); LOG.info("***********************************************************************************"); LOG.info("Checking FS permissions for root dir:" + rootDir); LOG.info("***********************************************************************************"); LOG.info(""); FileSystem fs = rootDir.getFileSystem(conf); short expectedPerms = Short.valueOf(fsPerms, 8); assertEquals( FsPermission.createImmutable(expectedPerms), fs.getFileStatus(rootDir).getPermission()); LOG.info("Checking FS permissions: SUCCESS"); }
@Override public FileStatus getFileStatus(Path path) throws IOException { File file = pathToFile(path); if (!file.exists()) { throw new FileNotFoundException("Can't find " + path); } // get close enough short mod = 0; if (file.canRead()) { mod |= 0444; } if (file.canWrite()) { mod |= 0200; } if (file.canExecute()) { mod |= 0111; } ShimLoader.getHadoopShims(); return new FileStatus(file.length(), file.isDirectory(), 1, 1024, file.lastModified(), file.lastModified(), FsPermission.createImmutable(mod), "owen", "users", path); } }
@Override public FileStatus getFileStatus(Path path) throws IOException { File file = pathToFile(path); if (!file.exists()) { throw new FileNotFoundException("Can't find " + path); } // get close enough short mod = 0; if (file.canRead()) { mod |= 0444; } if (file.canWrite()) { mod |= 0200; } if (file.canExecute()) { mod |= 0111; } return new FileStatus(file.length(), file.isDirectory(), 1, 1024, file.lastModified(), file.lastModified(), FsPermission.createImmutable(mod), "owen", "users", path); } }
@Override public FileStatus getFileStatus(Path path) throws IOException { File file = pathToFile(path); if (!file.exists()) { throw new FileNotFoundException("Cannot find " + path); } // get close enough short mod = 0; if (file.canRead()) { mod |= 0444; } if (file.canWrite()) { mod |= 0200; } if (file.canExecute()) { mod |= 0111; } return new FileStatus(file.length(), file.isDirectory(), 1, 1024, file.lastModified(), file.lastModified(), FsPermission.createImmutable(mod), "owen", "users", path); } }
@Override public FileStatus getFileStatus(Path path) throws IOException { File file = pathToFile(path); if (!file.exists()) { throw new FileNotFoundException("Can'table find " + path); } // get close enough short mod = 0; if (file.canRead()) { mod |= 0444; } if (file.canWrite()) { mod |= 0200; } if (file.canExecute()) { mod |= 0111; } return new FileStatus(file.length(), file.isDirectory(), 1, 1024, file.lastModified(), file.lastModified(), FsPermission.createImmutable(mod), "owen", "users", path); } }
@Override public FileStatus getFileStatus(Path path) throws IOException { File file = pathToFile(path); if (!file.exists()) { throw new FileNotFoundException("Can't find " + path); } // get close enough short mod = 0; if (file.canRead()) { mod |= 0444; } if (file.canWrite()) { mod |= 0200; } if (file.canExecute()) { mod |= 0111; } return new FileStatus(file.length(), file.isDirectory(), 1, 1024, file.lastModified(), file.lastModified(), FsPermission.createImmutable(mod), "owen", "users", path); } }
@Test(expected = IOException.class) public void testBackupPathIsNotAccessible() throws Exception { Path path = new Path(PERMISSION_TEST_PATH); FileSystem rootFs = FileSystem.get(TEST_UTIL.getConnection().getConfiguration()); rootFs.mkdirs(path.getParent()); rootFs.setPermission(path.getParent(), FsPermission.createImmutable((short) 000)); FileSystem fs = DFSTestUtil.getFileSystemAs(DIANA, TEST_UTIL.getConnection().getConfiguration()); fs.mkdirs(path); } }
FsPermission LOG_FILE_PERMISSION = FsPermission.createImmutable((short) 0777); Path logdir = new Path(conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + applicationAttemptID.getApplicationId().toString()
/** @throws Exception If failed. */ @SuppressWarnings("OctalInteger") @Test public void testMkdirs() throws Exception { Path fsHome = new Path(PRIMARY_URI); final Path dir = new Path(fsHome, "/tmp/staging"); final Path nestedDir = new Path(dir, "nested"); final FsPermission dirPerm = FsPermission.createImmutable((short)0700); final FsPermission nestedDirPerm = FsPermission.createImmutable((short)111); assertTrue(fs.mkdirs(dir, dirPerm)); assertTrue(fs.mkdirs(nestedDir, nestedDirPerm)); assertEquals(dirPerm, fs.getFileStatus(dir).getPermission()); assertEquals(nestedDirPerm, fs.getFileStatus(nestedDir).getPermission()); assertEquals(getClientFsUser(), fs.getFileStatus(dir).getOwner()); assertEquals(getClientFsUser(), fs.getFileStatus(nestedDir).getOwner()); }
@Test(expected = RuntimeException.class) public void testSplitGenFailure() throws IOException { Properties properties = new Properties(); HiveOutputFormat<?, ?> outFormat = new OrcOutputFormat(); org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter writer = outFormat.getHiveRecordWriter(conf, testFilePath, MyRow.class, true, properties, Reporter.NULL); writer.write(new OrcSerde().serialize(null,null)); writer.close(true); InputFormat<?,?> in = new OrcInputFormat(); fs.setPermission(testFilePath, FsPermission.createImmutable((short) 0333)); FileInputFormat.setInputPaths(conf, testFilePath.toString()); try { in.getSplits(conf, 1); } catch (RuntimeException e) { assertEquals(true, e.getMessage().contains("Permission denied")); throw e; } }
/** @throws Exception If failed. */ @SuppressWarnings("OctalInteger") @Test public void testMkdirs() throws Exception { Path fsHome = new Path(primaryFileSystemUriPath()); Path dir = new Path(fsHome, "/tmp/staging"); Path nestedDir = new Path(dir, "nested"); FsPermission dirPerm = FsPermission.createImmutable((short)0700); FsPermission nestedDirPerm = FsPermission.createImmutable((short)111); fs.mkdir(dir, dirPerm, true); fs.mkdir(nestedDir, nestedDirPerm, true); assertEquals(dirPerm, fs.getFileStatus(dir).getPermission()); assertEquals(nestedDirPerm, fs.getFileStatus(nestedDir).getPermission()); assertEquals(getClientFsUser(), fs.getFileStatus(dir).getOwner()); assertEquals(getClientFsUser(), fs.getFileStatus(nestedDir).getOwner()); }
@Test(expected=AclException.class) public void testModifyAclEntriesDefaultOnFile() throws IOException { fs.create(path).close(); fs.setPermission(path, FsPermission.createImmutable((short)0640)); List<AclEntry> aclSpec = Lists.newArrayList( aclEntry(DEFAULT, USER, "foo", ALL)); fs.modifyAclEntries(path, aclSpec); }
@Test public void testRemoveAclMinimalAcl() throws IOException { fs.create(path).close(); fs.setPermission(path, FsPermission.createImmutable((short)0640)); fs.removeAcl(path); AclStatus s = fs.getAclStatus(path); AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]); assertArrayEquals(new AclEntry[] { }, returned); assertPermission((short)0640); assertAclFeature(false); }