private void logProgress(String fileOffset, boolean prefixNewLine) throws IOException { long now = System.currentTimeMillis(); LogEntry entry = new LogEntry(now, componentID, fileOffset); String line = entry.toString(); if (prefixNewLine) { lockFileStream.writeBytes(System.lineSeparator() + line); } else { lockFileStream.writeBytes(line); } lockFileStream.hflush(); lastEntry = entry; // update this only after writing to hdfs }
@Override public void write(Writable w) throws IOException { Text t = (Text)w; os.writeBytes(t.toString()); os.writeBytes("\n"); }
private void createTextFile(Path file, int lineCount) throws IOException { try (FSDataOutputStream os = fs.create(file)) { for (int i = 0; i < lineCount; i++) { os.writeBytes("line " + i + System.lineSeparator()); } } }
private void createTextFile(Path file, int lineCount) throws IOException { FSDataOutputStream os = fs.create(file); int size = 0; for (int i = 0; i < lineCount; i++) { os.writeBytes("line " + i + System.lineSeparator()); String msg = "line " + i + System.lineSeparator(); size += msg.getBytes().length; } os.close(); }
@Override public FSDataOutputStream call() throws IOException { try { FileSystem fs = FSUtils.getCurrentFileSystem(this.conf); FsPermission defaultPerms = FSUtils.getFilePermissions(fs, this.conf, HConstants.DATA_FILE_UMASK_KEY); Path tmpDir = getTmpDir(conf); this.hbckLockPath = new Path(tmpDir, HBCK_LOCK_FILE); fs.mkdirs(tmpDir); final FSDataOutputStream out = createFileWithRetries(fs, this.hbckLockPath, defaultPerms); out.writeBytes(InetAddress.getLocalHost().toString()); // Add a note into the file we write on why hbase2 is writing out an hbck1 lock file. out.writeBytes(" Written by an hbase-2.x Master to block an " + "attempt by an hbase-1.x HBCK tool making modification to state. " + "See 'HBCK must match HBase server version' in the hbase refguide."); out.flush(); return out; } catch(RemoteException e) { if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){ return null; } else { throw e; } } }
private Path createFile(FileSystem fs, FsPermission perms) throws IOException { StringBuilder buf = new StringBuilder(); for (int i = 0; i < 10; i++) { buf.append((char)(rand.nextInt(26) + 'a')); } Path p = new Path(buf.toString()); FSDataOutputStream os = fs.create(p); os.writeBytes("Mary had a little lamb\nit's fleece was white as snow\nand anywhere that Mary " + "went\nthe lamb was sure to go\n"); os.close(); fs.setPermission(p, perms); fs.deleteOnExit(p); return p; }
fout.writeBytes(elem.valueToString() + "\n");
private FileStatus[] addMockStoreFiles(int count, MasterServices services, Path storedir) throws IOException { // get the existing store files FileSystem fs = services.getMasterFileSystem().getFileSystem(); fs.mkdirs(storedir); // create the store files in the parent for (int i = 0; i < count; i++) { Path storeFile = new Path(storedir, "_store" + i); FSDataOutputStream dos = fs.create(storeFile, true); dos.writeBytes("Some data: " + i); dos.close(); } LOG.debug("Adding " + count + " store files to the storedir:" + storedir); // make sure the mock store files are there FileStatus[] storeFiles = fs.listStatus(storedir); assertEquals("Didn't have expected store files", count, storeFiles.length); return storeFiles; }
/** @throws Exception If failed. */ @Test public void testRenameIfSrcPathIsAlreadyBeingOpenedToWrite() throws Exception { Path fsHome = new Path(primaryFsUri); Path srcFile = new Path(fsHome, "srcFile"); Path dstFile = new Path(fsHome, "dstFile"); FSDataOutputStream os = fs.create(srcFile); os.close(); os = fs.append(srcFile); assertTrue(fs.rename(srcFile, dstFile)); assertPathExists(fs, dstFile); String testStr = "Test"; try { os.writeBytes(testStr); } finally { os.close(); } try (FSDataInputStream is = fs.open(dstFile)) { byte[] buf = new byte[testStr.getBytes().length]; is.readFully(buf); assertEquals(testStr, new String(buf)); } }
FSDataOutputStream hostLocalOut = fs.create(hostLocalPath); fs.setPermission(hostLocalPath, new FsPermission(LOG_FILE_PERMISSION)); hostLocalOut.writeBytes(containerHostnames.toString().substring(1, containerHostnames.toString().length() - 1)); hostLocalOut.close(); LOG.info("host local enable is true, write " + hostLocalPath.toString() + " success"); logMessage.put(AMParams.PS_MEMORY, String.format("%.2f", psMemory / 1024.0)); out.writeBytes(new Gson().toJson(logMessage)); out.close(); fs.close();
RecordIdentifier ri = new RecordIdentifier(maxTxn - 1, bucket, i); ri.write(out); out.writeBytes("mary had a little lamb its fleece was white as snow\n");
FSDataOutputStream os = fs.create(file); for (int i = 0; i < 10; i++) { os.writeBytes("mary had a little lamb its fleece was white as snow\n");
FSDataOutputStream os = fs.create(file); for (int i = 0; i < 10; i++) { os.writeBytes("mary had a little lamb its fleece was white as snow\n");
/** @throws Exception If failed. */ @Test public void testRenameIfSrcPathIsAlreadyBeingOpenedToWrite() throws Exception { Path fsHome = new Path(primaryFsUri); Path srcFile = new Path(fsHome, "srcFile"); Path dstFile = new Path(fsHome, "dstFile"); FSDataOutputStream os = fs.create(srcFile, EnumSet.noneOf(CreateFlag.class), Options.CreateOpts.perms(FsPermission.getDefault())); os.close(); os = fs.create(srcFile, EnumSet.of(CreateFlag.APPEND), Options.CreateOpts.perms(FsPermission.getDefault())); fs.rename(srcFile, dstFile); assertPathExists(fs, dstFile); String testStr = "Test"; try { os.writeBytes(testStr); } finally { os.close(); } try (FSDataInputStream is = fs.open(dstFile)) { byte[] buf = new byte[testStr.getBytes().length]; is.readFully(buf); assertEquals(testStr, new String(buf)); } }
FSDataOutputStream out = miniDfs.getFileSystem().create( new Path("/path/to/schema/schema.avsc")); out.writeBytes(RECORD_SCHEMA); out.close(); String onHDFS = miniDfs.getFileSystem().getUri() + "/path/to/schema/schema.avsc";
/** * Write the split values to a tmp directory with unique name. Given that it is not known if the * supplied splits will be textual or binary, all splits will be encoded to enable proper handling * of binary data. */ private void writeSplitsToFileSystem(final FSDataOutputStream stream, final List<ByteBuffer> arguments, final int splitCount, final int splitOffset) throws IOException { for (int i = splitOffset; i < splitCount + splitOffset; i++) { byte[] splitBytes = ByteBufferUtil.toBytes(arguments.get(i)); String encodedSplit = Base64.getEncoder().encodeToString(splitBytes); stream.writeBytes(encodedSplit + '\n'); } }
@Override public Object apply(List<Object> args, Context context) throws ParseException { String content = (String)args.get(0); if(content == null) { return false; } String path = (String) args.get(1); if(path == null) { return false; } try(FSDataOutputStream os = fs.create(new Path(path))) { os.writeBytes(content); os.flush(); return true; } catch (IOException e) { String message = "Unable to write " + path + ": " + e.getMessage(); LOG.error(message, e); return false; } } }
private Path createTempFile(String filename, String contents) throws IOException { Path path = new Path(testRootDir, filename); FSDataOutputStream os = localFs.create(path); os.writeBytes(contents); os.close(); localFs.setPermission(path, new FsPermission("700")); return path; }
/** * Write the SortedSet of Tablet Directory names to the file system for use in the next phase of * the FATE operation. */ private void writeTabletDirectoriesToFileSystem(Master master, SortedSet<Text> dirs) throws IOException { FileSystem fs = master.getFileSystem().getDefaultVolume().getFileSystem(); if (fs.exists(new Path(tableInfo.getSplitDirsFile()))) fs.delete(new Path(tableInfo.getSplitDirsFile()), true); try (FSDataOutputStream stream = master.getOutputStream(tableInfo.getSplitDirsFile())) { for (Text dir : dirs) stream.writeBytes(dir + "\n"); } }
fsout.writeBytes(InetAddress.getLocalHost().getHostName()); fsout.hflush();