public static DelegationTokenInformation decodeDelegationTokenInformation(byte[] tokenBytes) throws IOException { DataInputStream in = new DataInputStream(new ByteArrayInputStream(tokenBytes)); DelegationTokenInformation token = new DelegationTokenInformation(0, null); int len = WritableUtils.readVInt(in); token.password = new byte[len]; in.readFully(token.password); token.renewDate = in.readLong(); return token; }
public static byte[] encodeDelegationTokenInformation(DelegationTokenInformation token) { try { ByteArrayOutputStream bos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(bos); WritableUtils.writeVInt(out, token.password.length); out.write(token.password); out.writeLong(token.renewDate); out.flush(); return bos.toByteArray(); } catch (IOException ex) { throw new RuntimeException("Failed to encode token.", ex); } }
@Override public void write(DataOutput out) throws IOException { WritableUtils.writeVInt(out, id); WritableUtils.writeVLong(out, expirationDate); if (secret == null) { WritableUtils.writeVInt(out, -1); } else { byte[] keyBytes = secret.getEncoded(); WritableUtils.writeVInt(out, keyBytes.length); out.write(keyBytes); } }
@Override public void write(DataOutput out) throws IOException { WritableUtils.writeEnum(out, primitiveCategory); WritableUtils.writeString(out, typeName); WritableUtils.writeString(out, primitiveJavaType.getName()); WritableUtils.writeString(out, primitiveJavaClass.getName()); WritableUtils.writeString(out, primitiveWritableClass.getName()); }
@Override public void readFields(DataInput in) throws IOException { primitiveCategory = WritableUtils.readEnum(in, PrimitiveObjectInspector.PrimitiveCategory.class); typeName = WritableUtils.readString(in); try { primitiveJavaType = Class.forName(WritableUtils.readString(in)); primitiveJavaClass = Class.forName(WritableUtils.readString(in)); primitiveWritableClass = Class.forName(WritableUtils.readString(in)); } catch (ClassNotFoundException e) { throw new IOException(e); } }
@Test public void testConsistencyWithHadoopVLong() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); for (long l : testNumbers) { baos.reset(); ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH); ByteBufferUtils.writeVLong(b, l); String bufStr = Bytes.toStringBinary(b.array(), b.arrayOffset(), b.position()); WritableUtils.writeVLong(dos, l); String baosStr = Bytes.toStringBinary(baos.toByteArray()); assertEquals(baosStr, bufStr); } }
public void writeWritable(Writable w) throws IOException { DataOutputStream dos = null; try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); dos = new DataOutputStream(baos); WritableUtils.writeString(dos, w.getClass().getName()); w.write(dos); out.writeBytes(baos.toByteArray(), Type.WRITABLE.code); dos.close(); dos = null; } finally { IOUtils.closeStream(dos); } }
String errMsg = "Server IPC version " + CURRENT_VERSION + " cannot communicate with client version " + clientVersion; ByteArrayOutputStream buffer = new ByteArrayOutputStream(); } else if (clientVersion == 2) { // Hadoop 0.18.3 RpcCall fakeCall = new RpcCall(this, 0); DataOutputStream out = new DataOutputStream(buffer); out.writeInt(0); // call ID out.writeBoolean(true); // error WritableUtils.writeString(out, VersionMismatch.class.getName()); WritableUtils.writeString(out, errMsg); fakeCall.setResponse(ByteBuffer.wrap(buffer.toByteArray())); sendResponse(fakeCall);
@Override protected Long decodeUnchecked(byte[] b, int offset, int len) { DataInputStream dis = new DataInputStream(new ByteArrayInputStream(b, offset, len)); try { return WritableUtils.readVLong(dis); } catch (IOException e) { throw new ValueFormatException(e); } } }
/** * Setup response for the IPC Call on Fatal Error from a * client that is using old version of Hadoop. * The response is serialized using the previous protocol's response * layout. * * @param response buffer to serialize the response into * @param call {@link Call} to which we are setting up the response * @param rv return value for the IPC Call, if the call was successful * @param errorClass error class, if the the call failed * @param error error message, if the call failed * @throws IOException */ private void setupResponseOldVersionFatal(ByteArrayOutputStream response, RpcCall call, Writable rv, String errorClass, String error) throws IOException { final int OLD_VERSION_FATAL_STATUS = -1; response.reset(); DataOutputStream out = new DataOutputStream(response); out.writeInt(call.callId); // write call id out.writeInt(OLD_VERSION_FATAL_STATUS); // write FATAL_STATUS WritableUtils.writeString(out, errorClass); WritableUtils.writeString(out, error); call.setResponse(ByteBuffer.wrap(response.toByteArray())); }
public Writable readWritable(Writable writable) throws IOException { DataInputStream dis = null; try { ByteArrayInputStream bais = new ByteArrayInputStream(in.readBytes()); dis = new DataInputStream(bais); String className = WritableUtils.readString(dis); if (writable == null) { try { Class<? extends Writable> cls = conf.getClassByName(className) .asSubclass(Writable.class); writable = (Writable) ReflectionUtils.newInstance(cls, conf); } catch (ClassNotFoundException e) { throw new IOException(e); } } else if (!writable.getClass().getName().equals(className)) { throw new IOException("wrong Writable class given"); } writable.readFields(dis); dis.close(); dis = null; return writable; } finally { IOUtils.closeStream(dis); } }
writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C.")); writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow")); writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris")); ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining()); DataInputStream data_input = new DataInputStream(byte_input); memstoreTS = WritableUtils.readVLong(data_input); buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS));
@Override protected List<V> decodeUnchecked(byte[] b, int offset, int origLen) { DataInputStream dis = new DataInputStream(new ByteArrayInputStream(b, offset, origLen)); try { int len = WritableUtils.readVInt(dis); List<V> vl = new ArrayList<>(len); for (int i = 0; i < len; i++) { vl.add(read(dis)); } return vl; } catch (IOException e) { throw new ValueFormatException(e); } } }
public void readFields(DataInput in) throws IOException { this.taskid.readFields(in); setProgress(in.readFloat()); this.numSlots = in.readInt(); this.runState = WritableUtils.readEnum(in, State.class); this.diagnosticInfo = Text.readString(in); this.stateString = Text.readString(in); this.phase = WritableUtils.readEnum(in, Phase.class); this.startTime = in.readLong(); this.finishTime = in.readLong(); counters = new Counters(); this.includeCounters = in.readBoolean(); this.outputSize = in.readLong(); if (includeCounters) { counters.readFields(in); } nextRecordRange.readFields(in); }
/** * No protobuf encoding of raw sasl messages */ protected final void doRawSaslReply(SaslStatus status, Writable rv, String errorClass, String error) throws IOException { BufferChain bc; // In my testing, have noticed that sasl messages are usually // in the ballpark of 100-200. That's why the initial capacity is 256. try (ByteBufferOutputStream saslResponse = new ByteBufferOutputStream(256); DataOutputStream out = new DataOutputStream(saslResponse)) { out.writeInt(status.state); // write status if (status == SaslStatus.SUCCESS) { rv.write(out); } else { WritableUtils.writeString(out, errorClass); WritableUtils.writeString(out, error); } bc = new BufferChain(saslResponse.getByteBuffer()); } doRespond(() -> bc); }
public void readFields(DataInput in) throws IOException { splitClass = Text.readString(in); dataLength = in.readLong(); bytes.readFields(in); int len = WritableUtils.readVInt(in); locations = new String[len]; for(int i=0; i < len; ++i) { locations[i] = Text.readString(in); } }
/** Workhorse to check and write out compressed data/lengths */ private synchronized void writeBuffer(DataOutputBuffer uncompressedDataBuffer) throws IOException { deflateFilter.resetState(); buffer.reset(); deflateOut.write(uncompressedDataBuffer.getData(), 0, uncompressedDataBuffer.getLength()); deflateOut.flush(); deflateFilter.finish(); WritableUtils.writeVInt(out, buffer.getLength()); out.write(buffer.getData(), 0, buffer.getLength()); }
public ByteBuffer readResponse() throws IOException { int length = in.readInt(); if (firstResponse) { firstResponse = false; // pre-rpcv9 exception, almost certainly a version mismatch. if (length == -1) { in.readInt(); // ignore fatal/error status, it's fatal for us. throw new RemoteException(WritableUtils.readString(in), WritableUtils.readString(in)); } } if (length <= 0) { throw new RpcException("RPC response has invalid length"); } if (maxResponseLength > 0 && length > maxResponseLength) { throw new RpcException("RPC response exceeds maximum data length"); } ByteBuffer bb = ByteBuffer.allocate(length); in.readFully(bb.array()); return bb; }
@Override @InterfaceAudience.Private public void readFields(DataInput in) throws IOException { blockSize = in.readLong(); bytesPerChecksum = in.readInt(); writePacketSize = in.readInt(); replication = in.readShort(); fileBufferSize = in.readInt(); checksumType = WritableUtils.readEnum(in, DataChecksum.Type.class); storagepolicyId = in.readByte(); } }
public void write(DataOutput out) throws IOException { Text.writeString(out, splitClass); out.writeLong(dataLength); bytes.write(out); WritableUtils.writeVInt(out, locations.length); for(int i = 0; i < locations.length; i++) { Text.writeString(out, locations[i]); } }