/** @throws Exception If failed. */ @Test public void testZeroReplicationFactor() throws Exception { // This test doesn't make sense for any mode except of PRIMARY. if (mode == PRIMARY) { Path igfsHome = new Path(PRIMARY_URI); Path file = new Path(igfsHome, "someFile"); try (FSDataOutputStream out = fs.create(file, (short)0)) { out.write(new byte[1024 * 1024]); } IgniteFileSystem igfs = grid(0).fileSystem("igfs"); IgfsPath filePath = new IgfsPath("/someFile"); IgfsFile fileInfo = igfs.info(filePath); awaitPartitionMapExchange(); Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length()); assertEquals(1, locations.size()); IgfsBlockLocation location = F.first(locations); assertEquals(1, location.nodeIds().size()); } }
/** * Test task. * * @throws Exception If failed. */ @SuppressWarnings("ConstantConditions") @Test public void testTask() throws Exception { String arg = DICTIONARY[new Random(System.currentTimeMillis()).nextInt(DICTIONARY.length)]; generateFile(TOTAL_WORDS); Long genLen = igfs.info(FILE).length(); IgniteBiTuple<Long, Integer> taskRes = igfs.execute(new Task(), new IgfsStringDelimiterRecordResolver(" "), Collections.singleton(FILE), arg); assert F.eq(genLen, taskRes.getKey()); assert F.eq(TOTAL_WORDS, taskRes.getValue()); }
/** * Test task. * * @throws Exception If failed. */ @SuppressWarnings("ConstantConditions") @Test public void testTaskAsync() throws Exception { String arg = DICTIONARY[new Random(System.currentTimeMillis()).nextInt(DICTIONARY.length)]; generateFile(TOTAL_WORDS); Long genLen = igfs.info(FILE).length(); IgniteBiTuple<Long, Integer> taskRes = igfs.executeAsync(new Task(), new IgfsStringDelimiterRecordResolver(" "), Collections.singleton(FILE), arg).get(); assert F.eq(genLen, taskRes.getKey()); assert F.eq(TOTAL_WORDS, taskRes.getValue()); }
/** @throws Exception If failed. */ @Test public void testZeroReplicationFactor() throws Exception { // This test doesn't make sense for any mode except of PRIMARY. if (mode == PRIMARY) { Path igfsHome = new Path(primaryFsUri); Path file = new Path(igfsHome, "someFile"); try (FSDataOutputStream out = fs.create(file, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault()), Options.CreateOpts.repFac((short)1))) { out.write(new byte[1024 * 1024]); } IgniteFileSystem igfs = grid(0).fileSystem("igfs"); IgfsPath filePath = new IgfsPath("/someFile"); IgfsFile fileInfo = igfs.info(filePath); Collection<IgfsBlockLocation> locations = igfs.affinity(filePath, 0, fileInfo.length()); assertEquals(1, locations.size()); IgfsBlockLocation location = F.first(locations); assertEquals(1, location.nodeIds().size()); } }
/** * Convert IGFS file information into Hadoop file status. * * @param file IGFS file information. * @return Hadoop file status. */ private FileStatus convert(IgfsFile file) { return new FileStatus( file.length(), file.isDirectory(), dfltReplication, file.groupBlockSize(), file.modificationTime(), file.accessTime(), permission(file), file.property(IgfsUtils.PROP_USER_NAME, user), file.property(IgfsUtils.PROP_GROUP_NAME, "users"), convert(file.path())) { @Override public String toString() { return "FileStatus [path=" + getPath() + ", isDir=" + isDirectory() + ", len=" + getLen() + "]"; } }; }
IgniteUuid.randomUuid(), igfsCtx.configuration().getBlockSize(), status.length(), null, null,
/** * A copy constructor. All the fields are copied from the copied {@code igfsFile}, but the {@code groupBlockSize} * which is specified separately. * * @param igfsFile The file to copy. * @param grpBlockSize Group block size. */ public IgfsFileImpl(IgfsFile igfsFile, long grpBlockSize) { A.notNull(igfsFile, "igfsFile"); this.path = igfsFile.path(); this.fileId = igfsFile instanceof IgfsFileImpl ? ((IgfsFileImpl)igfsFile).fileId : IgniteUuid.randomUuid(); this.blockSize = igfsFile.blockSize(); this.len = igfsFile.length(); this.grpBlockSize = igfsFile.isFile() ? grpBlockSize : 0L; this.props = igfsFile.properties(); this.accessTime = igfsFile.accessTime(); this.modificationTime = igfsFile.modificationTime(); this.flags = IgfsUtils.flags(igfsFile.isDirectory(), igfsFile.isFile()); }
/** * Convert IGFS file information into Hadoop file status. * * @param file IGFS file information. * @return Hadoop file status. */ @SuppressWarnings("deprecation") private FileStatus convert(IgfsFile file) { return new FileStatus( file.length(), file.isDirectory(), getDefaultReplication(), file.groupBlockSize(), file.modificationTime(), file.accessTime(), permission(file), file.property(IgfsUtils.PROP_USER_NAME, user), file.property(IgfsUtils.PROP_GROUP_NAME, "users"), convert(file.path())) { @Override public String toString() { return "FileStatus [path=" + getPath() + ", isDir=" + isDir() + ", len=" + getLen() + ", mtime=" + getModificationTime() + ", atime=" + getAccessTime() + ']'; } }; }
/** * Calculates size of directory or file for given ID. * * @param file IGFS File object. * @param sum Summary object that will collect information. * @throws IgniteCheckedException If failed. */ private void summaryRecursive(IgfsFile file, IgfsPathSummary sum) throws IgniteCheckedException { assert file != null; assert sum != null; if (file.isDirectory()) { if (!F.eq(IgfsPath.ROOT, file.path())) sum.directoriesCount(sum.directoriesCount() + 1); for (IgfsFile childFile : listFiles(file.path())) summaryRecursive(childFile, sum); } else { sum.filesCount(sum.filesCount() + 1); sum.totalLength(sum.totalLength() + file.length()); } }
IgniteUuid.randomUuid(), igfsCtx.configuration().getBlockSize(), status.length(), affKey, createFileLockId(false),
/** @throws Exception If failed. */ @Test public void testCreateFileColocated() throws Exception { IgfsPath path = new IgfsPath("/colocated"); UUID uuid = UUID.randomUUID(); IgniteUuid affKey; long idx = 0; while (true) { affKey = new IgniteUuid(uuid, idx); if (grid(0).affinity(grid(0).igfsx("igfs").configuration().getDataCacheConfiguration() .getName()).mapKeyToNode(affKey).id().equals(grid(0).localNode().id())) break; idx++; } try (IgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) { // Write 5M, should be enough to test distribution. for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]); } IgfsFile info = fs.info(path); Collection<IgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length()); assertEquals(1, affNodes.size()); Collection<UUID> nodeIds = F.first(affNodes).nodeIds(); assertEquals(1, nodeIds.size()); assertEquals(grid(0).localNode().id(), F.first(nodeIds)); }
/** * @throws Exception If failed. */ @Test public void testAffinity() throws Exception { long fileSize = 32L * 1024 * 1024; IgfsPath filePath = new IgfsPath("/file"); try (OutputStream os = igfs.create(filePath, true)) { for(int i = 0; i < fileSize / chunk.length; ++i) os.write(chunk); } long len = igfs.info(filePath).length(); int start = 0; // Check default maxLen (maxLen = 0) for (int i = 0; i < igfs.context().data().groupBlockSize() / 1024; i++) { Collection<IgfsBlockLocation> blocks = igfs.affinity(filePath, start, len); assertEquals(F.first(blocks).start(), start); assertEquals(start + len, F.last(blocks).start() + F.last(blocks).length()); len -= 1024 * 2; start += 1024; } }
long fileLen = igfs.info(inFile).length();
/** * A copy constructor. All the fields are copied from the copied {@code igfsFile}, but the {@code groupBlockSize} * which is specified separately. * * @param igfsFile The file to copy. * @param grpBlockSize Group block size. */ public IgfsFileImpl(IgfsFile igfsFile, long grpBlockSize) { A.notNull(igfsFile, "igfsFile"); this.path = igfsFile.path(); this.fileId = igfsFile instanceof IgfsFileImpl ? ((IgfsFileImpl)igfsFile).fileId : IgniteUuid.randomUuid(); this.blockSize = igfsFile.blockSize(); this.len = igfsFile.length(); this.grpBlockSize = igfsFile.isFile() ? grpBlockSize : 0L; this.props = igfsFile.properties(); this.accessTime = igfsFile.accessTime(); this.modificationTime = igfsFile.modificationTime(); this.flags = IgfsUtils.flags(igfsFile.isDirectory(), igfsFile.isFile()); }
/** * Calculates size of directory or file for given ID. * * @param file IGFS File object. * @param sum Summary object that will collect information. * @throws IgniteCheckedException If failed. */ private void summaryRecursive(IgfsFile file, IgfsPathSummary sum) throws IgniteCheckedException { assert file != null; assert sum != null; if (file.isDirectory()) { if (!F.eq(IgfsPath.ROOT, file.path())) sum.directoriesCount(sum.directoriesCount() + 1); for (IgfsFile childFile : listFiles(file.path())) summaryRecursive(childFile, sum); } else { sum.filesCount(sum.filesCount() + 1); sum.totalLength(sum.totalLength() + file.length()); } }