public static void markCorrupted(Path rootdir, String logFileName, FileSystem fs) { Path file = new Path(getSplitLogDir(rootdir, logFileName), "corrupt"); try { fs.createNewFile(file); } catch (IOException e) { LOG.warn("Could not flag a log file as corrupted. Failed to create " + file, e); } }
@Override public boolean create(String storeName, String tableName) throws IOException { Path storePath = new Path(this.storeRootDir, storeName); if (!this.fs.exists(storePath) && !create(storeName)) { return false; } Path tablePath = new Path(storePath, tableName); if (this.fs.exists(tablePath)) { throw new IOException(String.format("State file %s already exists for table %s", tablePath, tableName)); } return this.fs.createNewFile(tablePath); }
public boolean createNewFile(Path f) throws java.io.IOException { return this.underlyingFs.createNewFile(replaceScheme(f, this.replacementScheme, this.underlyingScheme)); }
/** * Try locking the lock. * * @return <em>true</em> if the lock is successfully locked, * <em>false</em> if otherwise. * @throws JobLockException thrown if the {@link JobLock} fails to be acquired */ boolean tryLock(Path lockFile) throws JobLockException { log.debug("Attempting lock: {}", lockFile); try { return this.fs.createNewFile(lockFile); } catch (IOException e) { throw new JobLockException(e); } }
public SparkScoreIterationListener(int printIterations, String pathStr) { this.printIterations = printIterations; this.pathStr = pathStr; FileSystem nfs = CommonUtils.openHdfsConnect(); try { Path path = new Path(pathStr); if (!nfs.exists(path)) { nfs.createNewFile(path); } } catch (Exception e) { throw new RuntimeException(e); } finally { CommonUtils.closeHdfsConnect(nfs); } } @Override
/** * Acquire the lock. * * @throws JobLockException thrown if the {@link JobLock} fails to be acquired */ void lock(Path lockFile) throws JobLockException { log.debug("Creating lock: {}", lockFile); try { if (!this.fs.createNewFile(lockFile)) { throw new JobLockException("Failed to create lock file " + lockFile.getName()); } } catch (IOException e) { throw new JobLockException(e); } }
private void createDirectory(String partPath) throws IOException { Path part = new Path(partPath); fs.mkdirs(part); // create files under partitions to simulate real partitions fs.createNewFile(new Path(partPath + Path.SEPARATOR + "dummydata1")); fs.createNewFile(new Path(partPath + Path.SEPARATOR + "dummydata2")); fs.deleteOnExit(part); } }
@Override public void storeFile(final HRegionFileSystem region, final Path familyDir, final StoreFileInfo storeFile) throws IOException { Path referenceFile = new Path(familyDir, storeFile.getPath().getName()); boolean success = true; if (storeFile.isReference()) { // write the Reference object to the snapshot storeFile.getReference().write(workingDirFs, referenceFile); } else { // create "reference" to this store file. It is intentionally an empty file -- all // necessary information is captured by its fs location and filename. This allows us to // only figure out what needs to be done via a single nn operation (instead of having to // open and read the files as well). success = workingDirFs.createNewFile(referenceFile); } if (!success) { throw new IOException("Failed to create reference file:" + referenceFile); } } }
@Override public void close(boolean b) throws IOException { if (writer == null) { // we are closing a file without writing any data in it FileSystem fs = options.getFileSystem() == null ? path.getFileSystem(options.getConfiguration()) : options.getFileSystem(); fs.createNewFile(path); return; } writer.close(); }
private void createFile(String partPath, String filename) throws IOException { Path part = new Path(partPath); fs.mkdirs(part); fs.createNewFile(new Path(partPath + Path.SEPARATOR + filename)); fs.deleteOnExit(part); }
@Override public void close(boolean b) throws IOException { if (writer == null) { // we are closing a file without writing any data in it FileSystem fs = options.getFileSystem() == null ? path.getFileSystem(options.getConfiguration()) : options.getFileSystem(); fs.createNewFile(path); return; } writer.close(); }
@Override public void stop(CoprocessorEnvironment env) throws IOException { String fileName = null; if (env instanceof MasterCoprocessorEnvironment) { // if running on HMaster fileName = MASTER_FILE; } else if (env instanceof RegionServerCoprocessorEnvironment) { fileName = REGIONSERVER_FILE; } else if (env instanceof RegionCoprocessorEnvironment) { LOG.error("on RegionCoprocessorEnvironment!!"); } Configuration conf = UTIL.getConfiguration(); Path resultFile = new Path(UTIL.getDataTestDirOnTestFS(), fileName); FileSystem fs = FileSystem.get(conf); boolean result = fs.createNewFile(resultFile); LOG.info("create file " + resultFile + " return rc " + result); } }
private Path createTableFiles(final Path rootDir, final String tableName, final Set<String> tableRegions, final Set<String> tableFamilies, final Set<String> tableHFiles) throws IOException { Path tableDir = new Path(rootDir, tableName); for (int r = 0; r < 10; ++r) { String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r)); tableRegions.add(regionName); Path regionDir = new Path(tableDir, regionName); for (int f = 0; f < 3; ++f) { String familyName = "f" + f; tableFamilies.add(familyName); Path familyDir = new Path(regionDir, familyName); fs.mkdirs(familyDir); for (int h = 0; h < 5; ++h) { String hfileName = TEST_UTIL.getRandomUUID().toString().replaceAll("-", ""); tableHFiles.add(hfileName); fs.createNewFile(new Path(familyDir, hfileName)); } } } return tableDir; } }
@Test public void testCreationCases() throws IOException { TrashTestBase trash; // If trash ident file doesn't exist, but trash is empty, create trash = new TrashTestBase(new Properties(), true, false, true); verify(trash.fs).createNewFile(new Path(trash.trash.getTrashLocation(), Trash.TRASH_IDENTIFIER_FILE)); // If trash ident file doesn't exist, but trash is not empty, fail try { trash = new TrashTestBase(new Properties(), true, false, false); Assert.fail(); } catch(IOException ioe) {} trash = new TrashTestBase(new Properties(), false, false, true); verify(trash.fs).mkdirs(trash.trash.getTrashLocation(), new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE)); verify(trash.fs).createNewFile(new Path(trash.trash.getTrashLocation(), Trash.TRASH_IDENTIFIER_FILE)); }
@BeforeClass public void setUp() throws IOException { this.fs = FileSystem.getLocal(new Configuration()); this.fs.delete(new Path(ROOT_DIR), true); Path dir1 = new Path(ROOT_DIR, "dir1"); Path dir2 = new Path(ROOT_DIR, "dir2"); this.fs.mkdirs(dir1); this.fs.mkdirs(dir2); Path src = new Path(dir1, "file"); Path dst = new Path(dir2, "file"); this.fs.createNewFile(src); this.step = (FsRenameCommitStep) new FsRenameCommitStep.Builder<>().from(src).to(dst).withProps(new State()).build(); }
@Override public void triggerConflict(ConnectorSession session, SchemaTableName tableName, ConnectorInsertTableHandle insertTableHandle, List<PartitionUpdate> partitionUpdates) throws IOException { for (PartitionUpdate partitionUpdate : partitionUpdates) { if ("pk2=insert2".equals(partitionUpdate.getTargetPath().getName())) { path = new Path(partitionUpdate.getTargetPath(), partitionUpdate.getFileNames().get(0)); break; } } assertNotNull(path); context = new HdfsContext(session, tableName.getSchemaName(), tableName.getTableName()); FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, path); fileSystem.createNewFile(path); }
@Test public void testIsFileDeletable() throws IOException, ReplicationException { // 1. Create a file Path file = new Path(root, "testIsFileDeletableWithNoHFileRefs"); fs.createNewFile(file); // 2. Assert file is successfully created assertTrue("Test file not created!", fs.exists(file)); ReplicationHFileCleaner cleaner = new ReplicationHFileCleaner(); cleaner.setConf(conf); // 3. Assert that file as is should be deletable assertTrue("Cleaner should allow to delete this file as there is no hfile reference node " + "for it in the queue.", cleaner.isFileDeletable(fs.getFileStatus(file))); List<Pair<Path, Path>> files = new ArrayList<>(1); files.add(new Pair<>(null, file)); // 4. Add the file to hfile-refs queue rq.addHFileRefs(peerId, files); // 5. Assert file should not be deletable assertFalse("Cleaner should not allow to delete this file as there is a hfile reference node " + "for it in the queue.", cleaner.isFileDeletable(fs.getFileStatus(file))); }
assertTrue(fileSystem.createNewFile(new Path(location, "empty-file"))); assertEquals(listDirectory(context, location), ImmutableList.of("empty-file"));
@Test public void testTTLCleaner() throws IOException, InterruptedException { FileSystem fs = UTIL.getDFSCluster().getFileSystem(); Path root = UTIL.getDataTestDirOnTestFS(); Path file = new Path(root, "file"); fs.createNewFile(file); long createTime = System.currentTimeMillis(); assertTrue("Test file not created!", fs.exists(file)); TimeToLiveHFileCleaner cleaner = new TimeToLiveHFileCleaner(); // update the time info for the file, so the cleaner removes it fs.setTimes(file, createTime - 100, -1); Configuration conf = UTIL.getConfiguration(); conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, 100); cleaner.setConf(conf); assertTrue("File not set deletable - check mod time:" + getFileStats(file, fs) + " with create time:" + createTime, cleaner.isFileDeletable(fs.getFileStatus(file))); }
/** * Test old recovered edits file doesn't break WALSplitter. * This is useful in upgrading old instances. */ @Test public void testOldRecoveredEditsFileSidelined() throws IOException { byte [] encoded = RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(); Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME); Path regiondir = new Path(tdir, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); fs.mkdirs(regiondir); long now = System.currentTimeMillis(); Entry entry = new Entry(new WALKeyImpl(encoded, TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID), new WALEdit()); Path parent = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); assertEquals(HConstants.RECOVERED_EDITS_DIR, parent.getName()); fs.createNewFile(parent); // create a recovered.edits file Path p = WALSplitter.getRegionSplitEditsPath(entry, FILENAME_BEING_SPLIT, TMPDIRNAME, conf); String parentOfParent = p.getParent().getParent().getName(); assertEquals(parentOfParent, RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName()); WALFactory.createRecoveredEditsWriter(fs, p, conf).close(); }