/** * Creates a mock {@link FileInStream} which will supply the given bytes. * * @param context file system context * @param bytes the bytes to supply */ public MockFileInStream(FileSystemContext context, byte[] bytes, AlluxioConfiguration conf) { super(new URIStatus(new FileInfo()), new InStreamOptions(new URIStatus(new FileInfo()), conf), context); mStream = new ByteArrayInputStream(bytes); }
private FileInfo createFileWithBlocksOnWorkers(String testFile, int... workerInds) throws Exception { return createFileWithBlocksOnWorkers(testFile, new FileInfo(), workerInds); }
/** * Tests that the worker will not delete the source directory if the directory still contains * files. */ @Test public void dontDeleteNonEmptySourceTest() throws Exception { when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))) .thenReturn(Lists.newArrayList(new URIStatus(new FileInfo()))); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem, times(0)).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
/** * Tests that the worker will delete the source directory if the directory contains only * directories. */ @Test public void deleteDirsOnlySourceDir() throws Exception { String inner = TEST_DIR + "/innerDir"; when(mMockFileSystem.listStatus(new AlluxioURI(TEST_DIR))).thenReturn( Lists.newArrayList(new URIStatus(new FileInfo().setPath(inner).setFolder(true)))); when(mMockFileSystem.listStatus(new AlluxioURI(inner))) .thenReturn(Lists.<URIStatus>newArrayList()); runTask(TEST_DIR, TEST_SOURCE, TEST_DESTINATION, WriteType.THROUGH); verify(mMockFileSystem).delete(eq(new AlluxioURI(TEST_DIR)), any(DeletePOptions.class)); }
/** * Tests for the {@link BaseFileSystem#listStatus(AlluxioURI, ListStatusPOptions)} method. */ @Test public void listStatus() throws Exception { AlluxioURI file = new AlluxioURI("/file"); List<URIStatus> infos = new ArrayList<>(); infos.add(new URIStatus(new FileInfo())); ListStatusPOptions listStatusOptions = ListStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.listStatus(file, listStatusOptions)).thenReturn(infos); assertSame(infos, mFileSystem.listStatus(file, listStatusOptions)); verify(mFileSystemMasterClient).listStatus(file, listStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }
@Test public void getInStreamMissingBlock() throws Exception { URIStatus dummyStatus = new URIStatus( new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(NotFoundException.class); mException.expectMessage("unavailable in both Alluxio and UFS"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
/** * Tests for the {@link BaseFileSystem#openFile(AlluxioURI, OpenFilePOptions)} method to * complete successfully. */ @Test public void openFile() throws Exception { AlluxioURI file = new AlluxioURI("/file"); URIStatus status = new URIStatus(new FileInfo()); GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); verify(mFileSystemMasterClient).getStatus(file, getStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }
/** * Tests for the {@link BaseFileSystem#getStatus(AlluxioURI, GetStatusPOptions)} method. */ @Test public void getStatus() throws Exception { AlluxioURI file = new AlluxioURI("/file"); URIStatus status = new URIStatus(new FileInfo()); GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); assertSame(status, mFileSystem.getStatus(file, getStatusOptions)); verify(mFileSystemMasterClient).getStatus(file, getStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }
@Test public void getInStreamNoWorkers() throws Exception { URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(UnavailableException.class); mException.expectMessage("No Alluxio worker available"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
private FileInfo createFileWithNoLocations(String testFile, int numOfBlocks) throws Exception { FileInfo testFileInfo = new FileInfo(); AlluxioURI uri = new AlluxioURI(testFile); List<FileBlockInfo> blockInfos = Lists.newArrayList(); for (int i = 0; i < numOfBlocks; i++) { blockInfos.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.<BlockLocation>newArrayList()))); } testFileInfo.setFolder(false).setPath(testFile).setFileBlockInfos(blockInfos); Mockito.when(mMockFileSystem.listStatus(uri)) .thenReturn(Lists.newArrayList(new URIStatus(testFileInfo))); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); return testFileInfo; } }
/** * Creates a directory with the given name. * * @return file info for the created directory */ private FileInfo createDirectory(String name) throws Exception { // Call all directories mount points to force cross-mount functionality. FileInfo info = new FileInfo().setFolder(true).setPath(name).setMountPoint(true); when(mMockFileSystem.getStatus(new AlluxioURI(name))).thenReturn(new URIStatus(info)); return info; }
/** * Tests the creation of a file via the * {@link BaseFileSystem#createFile(AlluxioURI, CreateFilePOptions)} method. */ @Test public void createFile() throws Exception { doNothing().when(mFileSystemMasterClient) .createFile(any(AlluxioURI.class), any(CreateFilePOptions.class)); URIStatus status = new URIStatus(new FileInfo()); AlluxioURI file = new AlluxioURI("/file"); GetStatusPOptions getStatusOptions = GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).build(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); FileOutStream out = mFileSystem.createFile(file, CreateFilePOptions.getDefaultInstance()); verify(mFileSystemMasterClient).createFile(file, CreateFilePOptions.getDefaultInstance()); assertEquals(out.mUri, file); verifyFilesystemContextAcquiredAndReleased(); }
@Test public void getInStreamRemote() throws Exception { WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1"); WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2"); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); // We should sometimes get remote1 and sometimes get remote2. Set<WorkerNetAddress> results = new HashSet<>(); for (int i = 0; i < 40; i++) { results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); } assertEquals(Sets.newHashSet(remote1, remote2), results); }
@Test public void scheduleAsyncPersist() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); long blockId = 0; long workerId = 1; long fileId = 2; List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location = new BlockLocation().setWorkerId(workerId); blockInfoList.add(new FileBlockInfo().setBlockInfo( new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location)))); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getPath(fileId)).thenReturn(path); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); handler.scheduleAsyncPersistence(path); List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId); assertEquals(1, persistFiles.size()); assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds()); }
@Test public void getInStreamLocal() throws Exception { WorkerNetAddress remote = new WorkerNetAddress().setHost("remote"); WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL); // Mock away gRPC usage. OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build(); when(mWorkerClient.openLocalBlock(any(StreamObserver.class))) .thenAnswer(new Answer() { public Object answer(InvocationOnMock invocation) { StreamObserver<OpenLocalBlockResponse> observer = invocation.getArgumentAt(0, StreamObserver.class); observer.onNext(response); observer.onCompleted(); return mStreamObserver; } }); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays .asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local))); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions( new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)) .getAddress()); }
@Test public void selectExecutorsMissingLocationTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); try { new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(new WorkerInfo()), mMockJobMasterContext); } catch (Exception e) { Assert.assertEquals("Block " + blockId + " does not exist", e.getMessage()); } } }
/** * Tests the persistence of file with block on multiple workers. */ @Test public void persistenceFileWithBlocksOnMultipleWorkers() throws Exception { DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster)); AlluxioURI path = new AlluxioURI("/test"); List<FileBlockInfo> blockInfoList = new ArrayList<>(); BlockLocation location1 = new BlockLocation().setWorkerId(1); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location1)))); BlockLocation location2 = new BlockLocation().setWorkerId(2); blockInfoList.add(new FileBlockInfo() .setBlockInfo(new BlockInfo().setLocations(Lists.newArrayList(location2)))); long fileId = 2; when(mFileSystemMaster.getFileId(path)).thenReturn(fileId); when(mFileSystemMaster.getFileInfo(fileId)) .thenReturn(new FileInfo().setLength(1).setCompleted(true)); when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList); // no persist scheduled on any worker assertEquals(0, handler.pollFilesToPersist(1).size()); assertEquals(0, handler.pollFilesToPersist(2).size()); }
@Test public void getStatus() throws Exception { FileInfo fileInfo = new FileInfo() .setLastModificationTimeMs(111L) .setFolder(false) .setOwner("user1") .setGroup("group1") .setMode(00755); Path path = new Path("/dir"); alluxio.client.file.FileSystem alluxioFs = mock(alluxio.client.file.FileSystem.class); when(alluxioFs.getStatus(new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)))) .thenReturn(new URIStatus(fileInfo)); FileSystem alluxioHadoopFs = new FileSystem(alluxioFs); FileStatus fileStatus = alluxioHadoopFs.getFileStatus(path); assertFileInfoEqualsFileStatus(fileInfo, fileStatus); }
@Test public void selectExecutorsTest() throws Exception { AlluxioURI uri = new AlluxioURI("/test"); PersistConfig config = new PersistConfig(uri.getPath(), -1, true, ""); WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10); WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress); long blockId = 1; BlockInfo blockInfo = new BlockInfo().setBlockId(blockId); FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo); BlockLocation location = new BlockLocation(); location.setWorkerAddress(workerNetAddress); blockInfo.setLocations(Lists.newArrayList(location)); FileInfo testFileInfo = new FileInfo(); testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo)); Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo)); Map<WorkerInfo, SerializableVoid> result = new PersistDefinition(mMockFileSystemContext, mMockFileSystem).selectExecutors(config, Lists.newArrayList(workerInfo), mMockJobMasterContext); Assert.assertEquals(1, result.size()); Assert.assertEquals(workerInfo, result.keySet().iterator().next()); }
@Test public void getInStreamUfs() throws Exception { WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1"); WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2"); BlockInfo info = new BlockInfo().setBlockId(0); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); OpenFilePOptions readOptions = OpenFilePOptions.newBuilder() .setFileReadLocationPolicy(MockFileWriteLocationPolicy.class.getTypeName()).build(); InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf); ((MockFileWriteLocationPolicy) options.getUfsReadLocationPolicy()) .setHosts(Arrays.asList(worker1, worker2)); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn( Arrays.asList(new WorkerInfo().setAddress(worker1), new WorkerInfo().setAddress(worker2))); // Location policy chooses worker1 first. assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); // Location policy chooses worker2 second. assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress()); }