@Override public boolean isFile() throws IOException { return fs.isFile(file); }
@Override public boolean accept(Path p) { try { if (!fs.isFile(p)) { return false; } String name = p.getName(); return !name.startsWith("_") && !name.startsWith("."); } catch (IOException e) { throw new RuntimeException(e); } } };
@Override public boolean isFile(String path) throws IOException { FileSystem hdfs = getFs(); return hdfs.isFile(new Path(path)); }
@Override public Collection<File> getChildren() throws IOException { if (fs.isFile(file)) return Collections.emptyList(); return Collections2.transform(Arrays.asList(fs.listStatus(file)), getAsFile); }
private <T> Optional<T> readFile(String type, Path path, JsonCodec<T> codec) { try { if (!metadataFileSystem.isFile(path)) { return Optional.empty(); } try (FSDataInputStream inputStream = metadataFileSystem.open(path)) { byte[] json = ByteStreams.toByteArray(inputStream); return Optional.of(codec.fromJson(json)); } } catch (Exception e) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not read " + type, e); } }
@Override protected long getResourceTimestampImpl(String resPath) throws IOException { Path p = getRealHDFSPath(resPath); if (!fs.exists(p) || !fs.isFile(p)) { return 0; } try { return fs.getFileStatus(p).getModificationTime(); } catch (Exception e) { throw new IOException("Put resource fail", e); } }
@Override protected boolean existsImpl(String resPath) throws IOException { Path p = getRealHDFSPath(resPath); return fs.exists(p) && fs.isFile(p); }
private void deleteMetadataDirectory(Path metadataDirectory) { try { Path schemaPath = new Path(metadataDirectory, PRESTO_SCHEMA_FILE_NAME); if (!metadataFileSystem.isFile(schemaPath)) { // if there is no schema file, assume this is not a database, partition or table return; } if (!metadataFileSystem.delete(metadataDirectory, true)) { throw new PrestoException(HIVE_METASTORE_ERROR, "Could not delete metadata directory"); } } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }
public boolean isFile(Path f) throws java.io.IOException { return this.underlyingFs.isFile(replaceScheme(f, this.replacementScheme, this.underlyingScheme)); }
private List<Path> getChildSchemaDirectories(Path metadataDirectory) { try { if (!metadataFileSystem.isDirectory(metadataDirectory)) { return ImmutableList.of(); } ImmutableList.Builder<Path> childSchemaDirectories = ImmutableList.builder(); for (FileStatus child : metadataFileSystem.listStatus(metadataDirectory)) { if (!child.isDirectory()) { continue; } Path childPath = child.getPath(); if (childPath.getName().startsWith(".")) { continue; } if (metadataFileSystem.isFile(new Path(childPath, PRESTO_SCHEMA_FILE_NAME))) { childSchemaDirectories.add(childPath); } } return childSchemaDirectories.build(); } catch (IOException e) { throw new PrestoException(HIVE_METASTORE_ERROR, e); } }
@Override protected RawResource getResourceImpl(String resPath) throws IOException { Path p = getRealHDFSPath(resPath); if (fs.exists(p) && fs.isFile(p)) { FileStatus fileStatus = fs.getFileStatus(p); if (fileStatus.getLen() == 0) { logger.warn("Zero length file: {}. ", p); } FSDataInputStream in = fs.open(p); long ts = fileStatus.getModificationTime(); return new RawResource(resPath, ts, in); } else { return null; } }
@Override // For pfile, calculate the checksum for use in testing public FileChecksum getFileChecksum(Path f) throws IOException { if (scheme.equalsIgnoreCase("pfile") && fs.isFile(f)) { return getPFileChecksum(f); } return fs.getFileChecksum(f); }
@Override public boolean accept(Path p) { boolean result = false; try { // Return files and only files that match the editfile names pattern. // There can be other files in this directory other than edit files. // In particular, on error, we'll move aside the bad edit file giving // it a timestamp suffix. See moveAsideBadEditsFile. Matcher m = EDITFILES_NAME_PATTERN.matcher(p.getName()); result = walFS.isFile(p) && m.matches(); // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX, // because it means splitwal thread is writting this file. if (p.getName().endsWith(RECOVERED_LOG_TMPFILE_SUFFIX)) { result = false; } // Skip SeqId Files if (isSequenceIdFile(p)) { result = false; } } catch (IOException e) { LOG.warn("Failed isFile check on {}", p, e); } return result; } });
/** * ls -r for debugging purposes */ public static void debugLsr(Configuration conf, Path p, ErrorReporter errors) throws IOException { if (!LOG.isDebugEnabled() || p == null) { return; } FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { // nothing return; } errors.print(p.toString()); if (fs.isFile(p)) { return; } if (fs.getFileStatus(p).isDirectory()) { FileStatus[] fss= fs.listStatus(p); for (FileStatus status : fss) { debugLsr(conf, status.getPath(), errors); } } } }
public String decrypt(final String cipheredText, final String passphrasePath, final FileSystem fs) throws IOException { Preconditions.checkNotNull(cipheredText); Preconditions.checkNotNull(passphrasePath); final Path path = new Path(passphrasePath); Preconditions.checkArgument(fs.exists(path), "File does not exist at " + passphrasePath); Preconditions .checkArgument(fs.isFile(path), "Passphrase path is not a file. " + passphrasePath); final FileStatus fileStatus = fs.getFileStatus(path); Preconditions.checkArgument(USER_READ_PERMISSION_ONLY.equals(fileStatus.getPermission()), "Passphrase file should only have read only permission on only user. " + passphrasePath); final Crypto crypto = new Crypto(); try (BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(path), Charset.defaultCharset()))) { final String passphrase = br.readLine(); final String decrypted = crypto.decrypt(cipheredText, passphrase); Preconditions.checkNotNull(decrypted, "Was not able to decrypt"); return decrypted; } } }
/** * Reads a log file and outputs its contents. * * @param conf HBase configuration relevant to this log file * @param p path of the log file to be read * @throws IOException IOException */ public void processFile(final Configuration conf, final Path p) throws IOException { FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { System.err.println("ERROR, file doesnt exist: " + p); return; } if (!fs.isFile(p)) { System.err.println(p + " is not a file"); return; } FileStatus logFile = fs.getFileStatus(p); if (logFile.getLen() == 0) { out.println("Zero length file: " + p); return; } out.println("Opening procedure state-log: " + p); ProcedureWALFile log = new ProcedureWALFile(fs, logFile); processProcedureWALFile(log); }
try { Path p = new Path(partInfo.getLocation()); if (p.getFileSystem(conf).isFile(p)) { sizeInBytes += p.getFileSystem(conf).getFileStatus(p).getLen(); } else {
private boolean isFile(FileSystem fs, Path rootPath) { for (int attempt = 1; attempt <= getMaxAttempts(); attempt++) { try { return fs.isFile(rootPath); } catch (IOException e) { logger.error("Error while calling isFile for path:" + rootPath.toString() + " Attempt: #" + attempt + "/" + getMaxAttempts()); if (getRetryDelayMs() > 0) { try { Thread.sleep(getRetryDelayMs()); } catch (InterruptedException ie) { logger.error("Fetcher is interrupted while wating to retry.", ie); } } } } throw new VoldemortException( "After retrying " + getMaxAttempts() + "times, can not get result from filesystem for isFile."); }
protected void open(Path dstPath, CompressionCodec codeC, CompressionType compType, Configuration conf, FileSystem hdfs) throws IOException { if (useRawLocalFileSystem) { if (hdfs instanceof LocalFileSystem) { hdfs = ((LocalFileSystem)hdfs).getRaw(); } else { logger.warn("useRawLocalFileSystem is set to true but file system " + "is not of type LocalFileSystem: " + hdfs.getClass().getName()); } } if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile(dstPath)) { outStream = hdfs.append(dstPath); } else { outStream = hdfs.create(dstPath); } writer = SequenceFile.createWriter(conf, outStream, serializer.getKeyClass(), serializer.getValueClass(), compType, codeC); registerCurrentStream(outStream, hdfs, dstPath); }
@Test public void testMovePath() throws IOException, URISyntaxException { String expected = "test"; ByteArrayOutputStream actual = new ByteArrayOutputStream(); Path src = new Path("/src/file.txt"); Path dst = new Path("/dst/file.txt"); FileSystem fs1 = Mockito.mock(FileSystem.class); Mockito.when(fs1.exists(src)).thenReturn(true); Mockito.when(fs1.isFile(src)).thenReturn(true); Mockito.when(fs1.getUri()).thenReturn(new URI("fs1:////")); Mockito.when(fs1.getFileStatus(src)).thenReturn(new FileStatus(1, false, 1, 1, 1, src)); Mockito.when(fs1.open(src)) .thenReturn(new FSDataInputStream(new SeekableFSInputStream(new ByteArrayInputStream(expected.getBytes())))); Mockito.when(fs1.delete(src, true)).thenReturn(true); FileSystem fs2 = Mockito.mock(FileSystem.class); Mockito.when(fs2.exists(dst)).thenReturn(false); Mockito.when(fs2.getUri()).thenReturn(new URI("fs2:////")); Mockito.when(fs2.getConf()).thenReturn(new Configuration()); Mockito.when(fs2.create(dst, false)).thenReturn(new FSDataOutputStream(actual, null)); try (ParallelRunner parallelRunner = new ParallelRunner(1, fs1)) { parallelRunner.movePath(src, fs2, dst, Optional.<String>absent()); } Assert.assertEquals(actual.toString(), expected); }