private void onAppend(ChannelHandlerContext ctx) throws IOException { writeContinueHeader(ctx); final String nnId = params.namenodeId(); final int bufferSize = params.bufferSize(); DFSClient dfsClient = newDfsClient(nnId, conf); OutputStream out = dfsClient.append(path, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); resp = new DefaultHttpResponse(HTTP_1_1, OK); resp.headers().set(CONTENT_LENGTH, 0); ctx.pipeline().replace(this, HdfsWriter.class.getSimpleName(), new HdfsWriter(dfsClient, out, resp)); }
@Override public FSDataOutputStream doCall(final Path p) throws IOException { return dfs.append(getPathName(p), bufferSize, flag, progress, statistics); } @Override
@Override public FSDataOutputStream doCall(final Path p) throws IOException { return dfs.append(getPathName(p), bufferSize, flag, progress, statistics, favoredNodes); } @Override
@Override public FSDataOutputStream doCall(final Path p) throws IOException { return dfs.append(getPathName(p), bufferSize, flag, progress, statistics); } @Override
@Override public FSDataOutputStream doCall(final Path p) throws IOException { return dfs.append(getPathName(p), bufferSize, flag, progress, statistics, favoredNodes); } @Override
@Override public FSDataOutputStream doCall(final Path p) throws IOException { return dfs.append(getPathName(p), bufferSize, flag, progress, statistics); } @Override
@Override public FSDataOutputStream doCall(final Path p) throws IOException { return dfs.append(getPathName(p), bufferSize, flag, progress, statistics, favoredNodes); } @Override
/** * Append to an existing HDFS file. * * @param src file name * @param buffersize buffer size * @param flag indicates whether to append data to a new block instead of * the last block * @param progress for reporting write-progress; null is acceptable. * @param statistics file system statistics; null is acceptable. * @return an output stream for writing into the file * * @see ClientProtocol#append(String, String, EnumSetWritable) */ public HdfsDataOutputStream append(final String src, final int buffersize, EnumSet<CreateFlag> flag, final Progressable progress, final FileSystem.Statistics statistics) throws IOException { final DFSOutputStream out = append(src, buffersize, flag, null, progress); return createWrappedOutputStream(out, statistics, out.getInitialLen()); }
/** * Append to an existing HDFS file. * * @param src file name * @param buffersize buffer size * @param flag indicates whether to append data to a new block instead of * the last block * @param progress for reporting write-progress; null is acceptable. * @param statistics file system statistics; null is acceptable. * @return an output stream for writing into the file * * @see ClientProtocol#append(String, String, EnumSetWritable) */ public HdfsDataOutputStream append(final String src, final int buffersize, EnumSet<CreateFlag> flag, final Progressable progress, final FileSystem.Statistics statistics) throws IOException { final DFSOutputStream out = append(src, buffersize, flag, null, progress); return createWrappedOutputStream(out, statistics, out.getInitialLen()); }
/** * Append to an existing HDFS file. * * @param src file name * @param buffersize buffer size * @param flag indicates whether to append data to a new block instead of * the last block * @param progress for reporting write-progress; null is acceptable. * @param statistics file system statistics; null is acceptable. * @return an output stream for writing into the file * * @see ClientProtocol#append(String, String, EnumSetWritable) */ public HdfsDataOutputStream append(final String src, final int buffersize, EnumSet<CreateFlag> flag, final Progressable progress, final FileSystem.Statistics statistics) throws IOException { final DFSOutputStream out = append(src, buffersize, flag, null, progress); return createWrappedOutputStream(out, statistics, out.getInitialLen()); }
/** This optional operation is not yet supported. */ public FSDataOutputStream append(Path f, int bufferSize, Progressable progress) throws IOException { DFSOutputStream op = (DFSOutputStream)dfs.append(getPathName(f), bufferSize, progress); return new FSDataOutputStream(op, statistics, op.getInitialLen()); }
private void onAppend(ChannelHandlerContext ctx) throws IOException { writeContinueHeader(ctx); final String nnId = params.namenodeId(); final int bufferSize = params.bufferSize(); DFSClient dfsClient = newDfsClient(nnId, conf); OutputStream out = dfsClient.append(path, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, OK); resp.headers().set(CONTENT_LENGTH, 0); ctx.pipeline().replace(this, HdfsWriter.class.getSimpleName(), new HdfsWriter(dfsClient, out, resp)); }
/** * Append to an existing HDFS file. * * @param src file name * @param buffersize buffer size * @param flag indicates whether to append data to a new block instead of the * last block * @param progress for reporting write-progress; null is acceptable. * @param statistics file system statistics; null is acceptable. * @param favoredNodes FavoredNodes for new blocks * @return an output stream for writing into the file * @see ClientProtocol#append(String, String, EnumSetWritable) */ public HdfsDataOutputStream append(final String src, final int buffersize, EnumSet<CreateFlag> flag, final Progressable progress, final FileSystem.Statistics statistics, final InetSocketAddress[] favoredNodes) throws IOException { final DFSOutputStream out = append(src, buffersize, flag, getFavoredNodesStr(favoredNodes), progress); return createWrappedOutputStream(out, statistics, out.getInitialLen()); }
/** * Append to an existing HDFS file. * * @param src file name * @param buffersize buffer size * @param flag indicates whether to append data to a new block instead of the * last block * @param progress for reporting write-progress; null is acceptable. * @param statistics file system statistics; null is acceptable. * @param favoredNodes FavoredNodes for new blocks * @return an output stream for writing into the file * @see ClientProtocol#append(String, String, EnumSetWritable) */ public HdfsDataOutputStream append(final String src, final int buffersize, EnumSet<CreateFlag> flag, final Progressable progress, final FileSystem.Statistics statistics, final InetSocketAddress[] favoredNodes) throws IOException { final DFSOutputStream out = append(src, buffersize, flag, getFavoredNodesStr(favoredNodes), progress); return createWrappedOutputStream(out, statistics, out.getInitialLen()); }
/** * Append to an existing HDFS file. * * @param src file name * @param buffersize buffer size * @param flag indicates whether to append data to a new block instead of the * last block * @param progress for reporting write-progress; null is acceptable. * @param statistics file system statistics; null is acceptable. * @param favoredNodes FavoredNodes for new blocks * @return an output stream for writing into the file * @see ClientProtocol#append(String, String, EnumSetWritable) */ public HdfsDataOutputStream append(final String src, final int buffersize, EnumSet<CreateFlag> flag, final Progressable progress, final FileSystem.Statistics statistics, final InetSocketAddress[] favoredNodes) throws IOException { final DFSOutputStream out = append(src, buffersize, flag, getFavoredNodesStr(favoredNodes), progress); return createWrappedOutputStream(out, statistics, out.getInitialLen()); }
private void onAppend(ChannelHandlerContext ctx) throws IOException { writeContinueHeader(ctx); final String nnId = params.namenodeId(); final int bufferSize = params.bufferSize(); DFSClient dfsClient = newDfsClient(nnId, conf); OutputStream out = dfsClient.append(path, bufferSize, EnumSet.of(CreateFlag.APPEND), null, null); DefaultHttpResponse resp = new DefaultHttpResponse(HTTP_1_1, OK); resp.headers().set(CONTENT_LENGTH, 0); ctx.pipeline().replace(this, HdfsWriter.class.getSimpleName(), new HdfsWriter(dfsClient, out, resp)); }
OutputStream out; try { out = append(src, conf.getInt("io.file.buffer.size", 4096), null); } catch (RemoteException re) { IOException e = re.unwrapRemoteException(AlreadyBeingCreatedException.class);
/** * Append to lazy persist file is denied. * @throws IOException */ @Test public void testAppendIsDenied() throws IOException { startUpCluster(true, -1); final String METHOD_NAME = GenericTestUtils.getMethodName(); Path path = new Path("/" + METHOD_NAME + ".dat"); makeTestFile(path, BLOCK_SIZE, true); try { client.append(path.toString(), BUFFER_LENGTH, EnumSet.of(CreateFlag.APPEND), null, null).close(); fail("Append to LazyPersist file did not fail as expected"); } catch (Throwable t) { LOG.info("Got expected exception ", t); } }
out = append(src, conf.getInt("io.file.buffer.size", 4096), null); } catch (RemoteException re) { IOException e = re.unwrapRemoteException(AlreadyBeingCreatedException.class);
@Override void prepare() throws Exception { final Path filePath = new Path(file); DFSTestUtil.createFile(dfs, filePath, BlockSize, DataNodes, 0); // append to the file and leave the last block under construction out = this.client.append(file, BlockSize, EnumSet.of(CreateFlag.APPEND), null, null); byte[] appendContent = new byte[100]; new Random().nextBytes(appendContent); out.write(appendContent); ((HdfsDataOutputStream) out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH)); LocatedBlocks blks = dfs.getClient() .getLocatedBlocks(file, BlockSize + 1); assertEquals(1, blks.getLocatedBlocks().size()); nodes = blks.get(0).getLocations(); oldBlock = blks.get(0).getBlock(); LocatedBlock newLbk = client.getNamenode().updateBlockForPipeline( oldBlock, client.getClientName()); newBlock = new ExtendedBlock(oldBlock.getBlockPoolId(), oldBlock.getBlockId(), oldBlock.getNumBytes(), newLbk.getBlock().getGenerationStamp()); }