@Override public void sync() throws IOException { fdos.hsync(); }
@Override public void flush() throws IOException { // TODO(calvin): This functionality should be restricted to select output streams. //JCP! ifdef HADOOP1 //JCP> mOut.sync(); //JCP! else // Note that, hsync() flushes out the data in client's user buffer all the way to the disk // device which may result in much slower performance than sync(). mOut.hsync(); //JCP! endif }
@Override public void hsync() throws IOException { if (out != null) { out.hsync(); } }
@Override public void sync() throws IOException { out.hflush(); out.hsync(); }
@Override protected void doSync() throws IOException { LOG.info("Attempting to sync all data to filesystem"); if (this.out instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } }
protected void syncStream(final FSDataOutputStream stream) throws IOException { if (useHsync) { stream.hsync(); } else { stream.hflush(); } }
@Override protected void doSync() throws IOException { avroWriter.flush(); LOG.debug("Attempting to sync all data to filesystem"); if (this.out instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } }
@Override public long flush() throws IOException { if (outStream == null) { throw new IllegalStateException("Writer is not open"); } if (syncOnFlush) { outStream.hsync(); } else { outStream.hflush(); } return outStream.getPos(); }
@Override public void sync(boolean forceSync) throws IOException { FSDataOutputStream fsdos = this.output; if (fsdos == null) { return; // Presume closed } fsdos.flush(); if (forceSync) { fsdos.hsync(); } else { fsdos.hflush(); } }
public static void saveModel(FileSystem fs, Model model ) throws Exception{ String json = null; if (model instanceof MultiLayerNetwork) { json = ((MultiLayerNetwork)model).getLayerWiseConfigurations().toJson(); } else if (model instanceof ComputationGraph) { json = ((ComputationGraph)model).getConfiguration().toJson(); } byte [] byts = json.getBytes(); FSDataOutputStream out = fs.create(new Path(modelPath)); out.write(byts); out.hsync(); fs.close(); } }
@Override public void doCommit(Long txId) throws IOException { if (this.rotationPolicy.mark(this.offset)) { rotateOutputFile(); this.offset = 0; this.rotationPolicy.reset(); } else { if (this.out instanceof HdfsDataOutputStream) { ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync(); } } }
private void flush0(CompletableFuture<Long> future, ByteArrayOutputStream buffer, boolean sync) { try { if (buffer.size() > 0) { out.write(buffer.getBuffer(), 0, buffer.size()); if (sync) { out.hsync(); } else { out.hflush(); } } future.complete(out.getPos()); } catch (IOException e) { future.completeExceptionally(e); return; } }
@Override public void iterationDone(Model model, int iteration, int epoch) { if (printIterations <= 0) { printIterations = 1; } String newScore = ""; if (iteration % printIterations == 0) { double score = model.score(); newScore += "Score at iteration {" + iteration + "} is {" + score + "}"; log.info(newScore); } FileSystem nfs = null; try { nfs = CommonUtils.openHdfsConnect(); Path path = new Path(pathStr); //although using append function isn't best ways, but currently it still solve the score log existing or not FSDataOutputStream out = nfs.append(path);//. .create(path); out.write(newScore.getBytes()); out.write("\n".getBytes()); out.hsync(); out.close(); CommonUtils.closeHdfsConnect(nfs); } catch (RemoteException e) { if (nfs != null) { CommonUtils.closeHdfsConnect(nfs); } } catch (Exception e) { throw new RuntimeException(e); } }
@Override public void sync(FSDataOutputStream outputStream) throws IOException { outputStream.hflush(); outputStream.hsync(); } })
@Override public void sync(FSDataOutputStream outputStream) throws IOException { outputStream.hflush(); outputStream.hsync(); ((HdfsDataOutputStream) outputStream).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } })
.hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); } else { this.out.hsync();
private void flush() throws IOException { if (output == null) { return; // Presume closed } output.flush(); // NOTE : the following API call makes sure that the data is flushed to disk on DataNodes (akin to POSIX fsync()) // See more details here : https://issues.apache.org/jira/browse/HDFS-744 output.hsync(); }
protected void syncStream(final FSDataOutputStream stream) throws IOException { if (useHsync) { stream.hsync(); } else { stream.hflush(); } }
protected void syncStream(final FSDataOutputStream stream) throws IOException { if (useHsync) { stream.hsync(); } else { stream.hflush(); } }
@Test public void testHsyncWithFlushEnabled() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); byte[] buffer = getRandomBytesArray(); final Path testFilePath = path(methodName.getMethodName()); try (FSDataOutputStream stream = getStreamAfterWrite(fs, testFilePath, buffer, true)) { stream.hsync(); validate(fs, testFilePath, buffer, true); } }