@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { ListStatusPOptions listOptions = ListStatusPOptions.newBuilder().setRecursive(true).build(); List<URIStatus> statuses = mFileSystem.listStatus(path, listOptions); if (statuses == null || statuses.size() == 0) { return; } getSizeInfo(path, statuses, cl.hasOption(READABLE_OPTION_NAME), cl.hasOption(SUMMARIZE_OPTION_NAME), cl.hasOption(MEMORY_OPTION_NAME)); }
/** * @param source an Alluxio URI * @param fileSystem the Alluxio file system * @return whether the URI is a file or a directory which contains files (including recursively) * @throws Exception if an unexpected exception occurs */ private static boolean hasFiles(AlluxioURI source, FileSystem fileSystem) throws Exception { Stack<AlluxioURI> dirsToCheck = new Stack<>(); dirsToCheck.add(source); while (!dirsToCheck.isEmpty()) { try { for (URIStatus status : fileSystem.listStatus(dirsToCheck.pop())) { if (!status.isFolder()) { return true; } dirsToCheck.push(new AlluxioURI(status.getPath())); } } catch (FileDoesNotExistException e) { // This probably means another worker has deleted the directory already, so we can probably // return false here. To be safe though, we will fall through and complete the search. } } return false; }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
/** * Informs the mock that the given fileInfos are children of the parent. */ private void setChildren(String parent, FileInfo... children) throws Exception { List<URIStatus> statuses = new ArrayList<>(); for (FileInfo child : children) { statuses.add(new URIStatus(child)); } when(mMockFileSystem.listStatus(new AlluxioURI(parent))) .thenReturn(Lists.newArrayList(statuses)); }
private void queueNonPersistedRecursive(URIStatus status, Queue<AlluxioURI> toPersist) throws AlluxioException, IOException { AlluxioURI uri = new AlluxioURI(status.getPath()); if (status.isFolder()) { List<URIStatus> statuses = mFileSystem.listStatus(uri); for (URIStatus s : statuses) { queueNonPersistedRecursive(s, toPersist); } } else if (!status.isPersisted()) { toPersist.add(uri); } }
/** * Reads the contents of a directory. * * @param path The FS path of the directory * @param buff The FUSE buffer to fill * @param filter FUSE filter * @param offset Ignored in alluxio-fuse * @param fi FileInfo data structure kept by FUSE * @return 0 on success, a negative value on error */ @Override public int readdir(String path, Pointer buff, FuseFillDir filter, @off_t long offset, FuseFileInfo fi) { final AlluxioURI turi = mPathResolverCache.getUnchecked(path); LOG.trace("readdir({}) [Alluxio: {}]", path, turi); try { final List<URIStatus> ls = mFileSystem.listStatus(turi); // standard . and .. entries filter.apply(buff, ".", null, 0); filter.apply(buff, "..", null, 0); for (final URIStatus file : ls) { filter.apply(buff, file.getName(), null, 0); } } catch (Throwable t) { LOG.error("Failed to read directory {}", path, t); return AlluxioFuseUtils.getErrorCode(t); } return 0; }
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
/** * Tells the mock to throw FileDoesNotExistException when the given path is queried. */ private void setPathToNotExist(String path) throws Exception { AlluxioURI uri = new AlluxioURI(path); when(mMockFileSystem.getStatus(uri)).thenThrow(new FileDoesNotExistException(uri)); when(mMockFileSystem.listStatus(uri)).thenThrow(new FileDoesNotExistException(uri)); } }
/** * Tests that the worker will delete the source directory if the directory contains nothing. */ @Test public void deleteEmptySourceDir() throws Exception { when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))) .thenReturn(Lists.<URIStatus>newArrayList()); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
private void applyOperation() throws IOException, AlluxioException { AlluxioURI uri = new AlluxioURI(String.format("%s/%s", mDir, UUID.randomUUID())); switch (mOperation) { case CreateEmptyFile: mFileSystem.createFile(uri).close(); break; case CreateAndDeleteEmptyFile: mFileSystem.createFile(uri).close(); mFileSystem.delete(uri); break; case CreateFile: try (FileOutStream file = mFileSystem.createFile(uri)) { file.write(mFiledata); } break; case ListStatus: mFileSystem.listStatus(new AlluxioURI(mDir)); break; default: throw new IllegalStateException("Unknown operation: " + mOperation); } } }
/** * Tests that the worker will delete the source directory if the directory contains only * directories. */ @Test public void deleteDirsOnlySourceDir() throws Exception { String inner = TEST_DIR + "/innerDir"; when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))).thenReturn( Lists.newArrayList(new URIStatus(new FileInfo().setPath(inner).setFolder(true)))); when(mMockFileSystem.listStatus(new AlluxioURI(inner))) .thenReturn(Lists.<URIStatus>newArrayList()); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
/** * Tests that the worker will not delete the source directory if the directory still contains * files. */ @Test public void dontDeleteNonEmptySourceTest() throws Exception { when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))) .thenReturn(Lists.newArrayList(new URIStatus(new FileInfo()))); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem, times(0)).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
private long[] countHelper(AlluxioURI path) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (!status.isFolder()) { return new long[]{ 1L, 0L, status.getLength() }; } long[] rtn = new long[]{ 0L, 1L, 0L }; List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(path); } catch (AlluxioException e) { throw new IOException(e.getMessage()); } for (URIStatus uriStatus : statuses) { long[] toAdd = countHelper(new AlluxioURI(uriStatus.getPath())); rtn[0] += toAdd[0]; rtn[1] += toAdd[1]; rtn[2] += toAdd[2]; } return rtn; }
/** * Loads a file or directory in Alluxio space, makes it resident in memory. * * @param filePath The {@link AlluxioURI} path to load into Alluxio memory * @throws AlluxioException when Alluxio exception occurs * @throws IOException when non-Alluxio exception occurs */ private void load(AlluxioURI filePath, int replication) throws AlluxioException, IOException, InterruptedException { URIStatus status = mFileSystem.getStatus(filePath); if (status.isFolder()) { List<URIStatus> statuses = mFileSystem.listStatus(filePath); for (URIStatus uriStatus : statuses) { AlluxioURI newPath = new AlluxioURI(uriStatus.getPath()); load(newPath, replication); } } else { Thread thread = JobGrpcClientUtils.createProgressThread(System.out); thread.start(); try { JobGrpcClientUtils.run(new LoadConfig(filePath.getPath(), replication), 3, mFsContext.getConf()); } finally { thread.interrupt(); } } System.out.println(filePath + " loaded"); }
/** * Returns a list of files at a given {@code path}. * * @param fs a {@link FileSystem} handler * @param path a path in alluxio file system * @return a list of strings representing the file names under the given path */ public static List<String> listFiles(FileSystem fs, String path) { try { List<URIStatus> statuses = fs.listStatus(new AlluxioURI(path)); List<String> res = new ArrayList<>(); for (URIStatus status : statuses) { res.add(status.getPath()); if (status.isFolder()) { res.addAll(listFiles(fs, status.getPath())); } } return res; } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception { FileInfo testFileInfo = new FileInfo(); AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int i = 0; i < numOfBlocks; i++) { blockInfos.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList()))); } testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); Mockito.when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; } }
/** * Tests for the {@link BaseFileSystem#listStatus(AlluxioURI, ListStatusPOptions)} method. */ @Test public void listStatus() throws Exception { AlluxioURI file = new AlluxioURI("/file"); List<URIStatus> infos = new ArrayList<>(); infos.add(new URIStatus(new FileInfo())); ListStatusPOptions listStatusOptions = ListStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.listStatus(file, listStatusOptions)).thenReturn(infos); assertSame(infos, mFileSystem.listStatus(file, listStatusOptions)); verify(mFileSystemMasterClient).listStatus(file, listStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }
/** * Creates a file with the given name and a block on each specified worker. Workers may be * repeated to give them multiple blocks. * * @param testFile the name of the file to create * @param fileInfo file info to apply to the created file * @param workerInds the workers to put blocks on, specified by their indices * @return file info for the created file */ private FileInfo createFileWithBlocksOnWorkers(String testFile, FileInfo fileInfo, int... workerInds) throws Exception { AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int workerInd : workerInds) { WorkerNetAddress address = JOB_WORKERS.get(workerInd).getAddress(); blockInfos.add(new FileBlockInfo().setBlockInfo(new BlockInfo() .setLocations(Lists.newArrayList(new BlockLocation().setWorkerAddress(address))))); } FileInfo testFileInfo = fileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; }
/** * Ensures that an exception is propagated correctly when listing the status. */ @Test public void listStatusException() throws Exception { AlluxioURI file = new AlluxioURI("/file"); when(mFileSystemMasterClient.listStatus(file, ListStatusPOptions.getDefaultInstance())) .thenThrow(EXCEPTION); try { mFileSystem.listStatus(file, ListStatusPOptions.getDefaultInstance()); fail(SHOULD_HAVE_PROPAGATED_MESSAGE); } catch (Exception e) { assertSame(EXCEPTION, e); } verifyFilesystemContextAcquiredAndReleased(); }
/** * Tests that the {@link AbstractFileSystem#listStatus(Path)} method uses * {@link URIStatus#getLastModificationTimeMs()} correctly. */ @Test public void listStatus() throws Exception { FileInfo fileInfo1 = new FileInfo() .setLastModificationTimeMs(111L) .setFolder(false) .setOwner("user1") .setGroup("group1") .setMode(00755); FileInfo fileInfo2 = new FileInfo() .setLastModificationTimeMs(222L) .setFolder(true) .setOwner("user2") .setGroup("group2") .setMode(00644); Path path = new Path("/dir"); alluxio.client.file.FileSystem alluxioFs = mock(alluxio.client.file.FileSystem.class); when(alluxioFs.listStatus(new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)))) .thenReturn(Lists.newArrayList(new URIStatus(fileInfo1), new URIStatus(fileInfo2))); FileSystem alluxioHadoopFs = new FileSystem(alluxioFs); FileStatus[] fileStatuses = alluxioHadoopFs.listStatus(path); assertFileInfoEqualsFileStatus(fileInfo1, fileStatuses[0]); assertFileInfoEqualsFileStatus(fileInfo2, fileStatuses[1]); alluxioHadoopFs.close(); }