@Override public InputStream openInputStream() throws IOException { final FileSystem fs = path.getFileSystem(config); return fs.open(path); }
/** Construct given a path and a {@code FileSystem}. */ public FsInput(Path path, FileSystem fileSystem) throws IOException { this.len = fileSystem.getFileStatus(path).getLen(); this.stream = fileSystem.open(path); }
private BufferedReader openStatusFile(String fname) throws IOException { Path p = new Path(statusdir, fname); FileSystem fs = p.getFileSystem(conf); BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(p))); return in; }
private TxnRecord readTxnRecord(Path path) throws IOException { FSDataInputStream inputStream = null; try { inputStream = this.options.fs.open(path); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); String line; if ((line = reader.readLine()) != null) { String[] fields = line.split(","); return new TxnRecord(Long.valueOf(fields[0]), fields[1], Long.valueOf(fields[2])); } } finally { if (inputStream != null) { inputStream.close(); } } return new TxnRecord(0, options.currentFile.toString(), 0); }
Path dir = PerformanceEvaluation.writeInputFile(HTU.getConfiguration(), opts, HTU.getDataTestDir()); FileSystem fs = FileSystem.get(HTU.getConfiguration()); Path p = new Path(dir, PerformanceEvaluation.JOB_INPUT_FILENAME); long len = fs.getFileStatus(p).getLen(); assertTrue(len > 0); byte [] content = new byte[(int)len]; FSDataInputStream dis = fs.open(p); try { dis.readFully(content); BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(content))); int count = 0; while (br.readLine() != null) { dis.close();
public static Optional<String> getMasterPassword(FileSystem fs, Path masterPasswordFile) { try (Closer closer = Closer.create()) { if (!fs.exists(masterPasswordFile) || fs.getFileStatus(masterPasswordFile).isDirectory()) { LOG.warn(masterPasswordFile + " does not exist or is not a file. Cannot decrypt any encrypted password."); return Optional.absent(); } InputStream in = closer.register(fs.open(masterPasswordFile)); return Optional.of(new LineReader(new InputStreamReader(in, Charsets.UTF_8)).readLine()); } catch (IOException e) { throw new RuntimeException("Failed to obtain master password from " + masterPasswordFile, e); } }
public GuidDatasetUrnStateStoreNameParser(FileSystem fs, Path jobStatestoreRootDir) throws IOException { this.fs = fs; this.sanitizedNameToDatasetURNMap = Maps.synchronizedBiMap(HashBiMap.<String, String>create()); this.versionIdentifier = new Path(jobStatestoreRootDir, StateStoreNameVersion.V1.getDatasetUrnNameMapFile()); if (this.fs.exists(versionIdentifier)) { this.version = StateStoreNameVersion.V1; try (InputStream in = this.fs.open(versionIdentifier)) { LineReader lineReader = new LineReader(new InputStreamReader(in, Charsets.UTF_8)); String shortenName = lineReader.readLine(); while (shortenName != null) { String datasetUrn = lineReader.readLine(); this.sanitizedNameToDatasetURNMap.put(shortenName, datasetUrn); shortenName = lineReader.readLine(); } } } else { this.version = StateStoreNameVersion.V0; } }
@Override public void connect() throws IOException { Preconditions.checkState(is == null, "Already connected"); try { LOG.debug("Connecting to {}", url); FileSystem fs = FileSystem.get(url.toURI(), conf); is = fs.open(new Path(url.toURI())); } catch (URISyntaxException e) { throw new IOException(e.toString()); } }
/** @throws Exception If failed. */ @Test public void testOpenIfPathIsAlreadyOpened() throws Exception { Path fsHome = new Path(primaryFsUri); Path file = new Path(fsHome, "someFile"); FSDataOutputStream os = fs.create(file); os.close(); FSDataInputStream is1 = fs.open(file); FSDataInputStream is2 = fs.open(file); is1.close(); is2.close(); }
private void corruptDataFile(final String file, final Configuration conf, final int addRemoveBytes) throws Exception { Path bPath = new Path(file); Path cPath = new Path(bPath.getParent(), bPath.getName() + ".corrupt"); FileSystem fs = bPath.getFileSystem(conf); FileStatus fileStatus = fs.getFileStatus(bPath); int len = addRemoveBytes == Integer.MIN_VALUE ? 0 : (int) fileStatus.getLen() + addRemoveBytes; byte[] buffer = new byte[len]; FSDataInputStream fdis = fs.open(bPath); fdis.readFully(0, buffer, 0, (int) Math.min(fileStatus.getLen(), buffer.length)); fdis.close(); FSDataOutputStream fdos = fs.create(cPath, true); fdos.write(buffer, 0, buffer.length); fdos.close(); fs.delete(bPath, false); fs.rename(cPath, bPath); }
private String getSha(final Path localFile) throws IOException, IllegalArgumentException { FileSystem localFs = FileSystem.getLocal(conf); FileStatus fileStatus = localFs.getFileStatus(localFile); String key = getKey(fileStatus); String sha256 = shaCache.getIfPresent(key); if (sha256 == null) { FSDataInputStream is = null; try { is = localFs.open(localFile); long start = System.currentTimeMillis(); sha256 = DigestUtils.sha256Hex(is); long end = System.currentTimeMillis(); LOG.info("Computed sha: {} for file: {} of length: {} in {} ms", sha256, localFile, LlapUtil.humanReadableByteCount(fileStatus.getLen()), end - start); shaCache.put(key, sha256); } finally { if (is != null) { is.close(); } } } return sha256; } public void setQueueName(String queueName) {
/** * Create a {@link RegionInfo} from the serialized version on-disk. * @param fs {@link FileSystem} that contains the Region Info file * @param regionDir {@link Path} to the Region Directory that contains the Info file * @return An {@link RegionInfo} instance gotten from the Region Info file. * @throws IOException if an error occurred during file open/read operation. */ public static RegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir) throws IOException { FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE)); try { return RegionInfo.parseFrom(in); } finally { in.close(); } }
@Override public AppendDictSlice readSlice(String directory, String sliceFileName) throws IOException { Path path = new Path(directory, sliceFileName); logger.trace("read slice from {}", path); try (FSDataInputStream input = fileSystem.open(path, BUFFER_SIZE)) { return AppendDictSlice.deserializeFrom(input); } }
/** * Returns an InputStream for the file using the owning filesystem, * or the default if none is given. * @param filename The filename to be opened * @throws IOException */ static InputStream openFromFS(String filename) throws IOException { Path p = new Path(filename); return p.getFileSystem(new Configuration()).open(p); }
private void corruptFile(Path p) throws IOException { String manifestName = p.getName(); // Rename the original region-manifest file Path newP = new Path(p.getParent(), manifestName + "1"); fs.rename(p, newP); // Create a new region-manifest file FSDataOutputStream out = fs.create(p); //Copy the first 25 bytes of the original region-manifest into the new one, //make it a corrupted region-manifest file. FSDataInputStream input = fs.open(newP); byte[] buffer = new byte[25]; int len = input.read(0, buffer, 0, 25); if (len > 1) { out.write(buffer, 0, len - 1); } out.close(); // Delete the original region-manifest fs.delete(newP); }
/** * Similiar to {@link #tar(FileSystem, Path, Path)} except the source and destination {@link FileSystem} can be different. * * @see #tar(FileSystem, Path, Path) */ public static void tar(FileSystem sourceFs, FileSystem destFs, Path sourcePath, Path destPath) throws IOException { try (FSDataOutputStream fsDataOutputStream = destFs.create(destPath); TarArchiveOutputStream tarArchiveOutputStream = new TarArchiveOutputStream( new GzipCompressorOutputStream(fsDataOutputStream), ConfigurationKeys.DEFAULT_CHARSET_ENCODING.name())) { FileStatus fileStatus = sourceFs.getFileStatus(sourcePath); if (sourceFs.isDirectory(sourcePath)) { dirToTarArchiveOutputStreamRecursive(fileStatus, sourceFs, Optional.<Path> absent(), tarArchiveOutputStream); } else { try (FSDataInputStream fsDataInputStream = sourceFs.open(sourcePath)) { fileToTarArchiveOutputStream(fileStatus, fsDataInputStream, new Path(sourcePath.getName()), tarArchiveOutputStream); } } } }
protected static Schema getSchemaFromFS(String schemaFSUrl, Configuration conf) throws IOException, URISyntaxException { FSDataInputStream in = null; FileSystem fs = null; try { fs = FileSystem.get(new URI(schemaFSUrl), conf); } catch (IOException ioe) { //return null only if the file system in schema is not recognized if (LOG.isDebugEnabled()) { String msg = "Failed to open file system for uri " + schemaFSUrl + " assuming it is not a FileSystem url"; LOG.debug(msg, ioe); } return null; } try { in = fs.open(new Path(schemaFSUrl)); Schema s = AvroSerdeUtils.getSchemaFor(in); return s; } finally { if(in != null) in.close(); } }
/** @throws Exception If failed. */ @Test public void testGetFileBlockLocations() throws Exception { Path igfsHome = new Path(PRIMARY_URI); Path file = new Path(igfsHome, "someFile"); try (OutputStream out = new BufferedOutputStream(fs.create(file, true, 1024 * 1024))) { byte[] data = new byte[128 * 1024]; for (int i = 0; i < 100; i++) out.write(data); out.flush(); } try (FSDataInputStream in = fs.open(file, 1024 * 1024)) { byte[] data = new byte[128 * 1024]; int read; do { read = in.read(data); } while (read > 0); } FileStatus status = fs.getFileStatus(file); int grpLen = 128 * 512 * 1024; int grpCnt = (int)((status.getLen() + grpLen - 1) / grpLen); BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen()); assertEquals(grpCnt, locations.length); }
final Path workingDirPath = new Path(workingDirPathStr); FileSystem fs; try { fs = workingDirPath.getFileSystem(config); Path indexZip = new Path(path.getParent(), "index.zip"); if (descriptorParts.length > 1) { Preconditions.checkState(descriptorParts.length <= 3 && ); indexZip = new Path( path.getParent(), StringUtils.format( final DataSegment dataSegment = mapper.readValue(fs.open(path), DataSegment.class); log.info("Found segment [%s] located at [%s]", dataSegment.getId(), indexZip); if (updateDescriptor) { log.info("Updating loadSpec in descriptor.json at [%s] with new path [%s]", path, pathWithoutScheme); mapper.writeValue(fs.create(path, true), dataSegment);
@Test public void testMovePath() throws IOException, URISyntaxException { String expected = "test"; ByteArrayOutputStream actual = new ByteArrayOutputStream(); Path src = new Path("/src/file.txt"); Path dst = new Path("/dst/file.txt"); FileSystem fs1 = Mockito.mock(FileSystem.class); Mockito.when(fs1.exists(src)).thenReturn(true); Mockito.when(fs1.isFile(src)).thenReturn(true); Mockito.when(fs1.getUri()).thenReturn(new URI("fs1:////")); Mockito.when(fs1.getFileStatus(src)).thenReturn(new FileStatus(1, false, 1, 1, 1, src)); Mockito.when(fs1.open(src)) .thenReturn(new FSDataInputStream(new SeekableFSInputStream(new ByteArrayInputStream(expected.getBytes())))); Mockito.when(fs1.delete(src, true)).thenReturn(true); FileSystem fs2 = Mockito.mock(FileSystem.class); Mockito.when(fs2.exists(dst)).thenReturn(false); Mockito.when(fs2.getUri()).thenReturn(new URI("fs2:////")); Mockito.when(fs2.getConf()).thenReturn(new Configuration()); Mockito.when(fs2.create(dst, false)).thenReturn(new FSDataOutputStream(actual, null)); try (ParallelRunner parallelRunner = new ParallelRunner(1, fs1)) { parallelRunner.movePath(src, fs2, dst, Optional.<String>absent()); } Assert.assertEquals(actual.toString(), expected); }