Refine search
/** * Serialize the shuffle port into a ByteBuffer for use later on. * @param port the port to be sent to the ApplciationMaster * @return the serialized form of the port. */ public static ByteBuffer serializeMetaData(int port) throws IOException { //TODO these bytes should be versioned DataOutputBuffer port_dob = new DataOutputBuffer(); port_dob.writeInt(port); return ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength()); }
public static byte[] toBytes(SubmitWorkInfo submitWorkInfo) throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); submitWorkInfo.write(dob); return dob.getData(); }
/** * A helper function to serialize the JobTokenIdentifier to be sent to the * ShuffleHandler as ServiceData. * @param jobToken the job token to be used for authentication of * shuffle data requests. * @return the serialized version of the jobToken. */ public static ByteBuffer serializeServiceData(Token<JobTokenIdentifier> jobToken) throws IOException { //TODO these bytes should be versioned DataOutputBuffer jobToken_dob = new DataOutputBuffer(); jobToken.write(jobToken_dob); return ByteBuffer.wrap(jobToken_dob.getData(), 0, jobToken_dob.getLength()); }
/** Convert writables to a byte array */ public static byte[] toByteArray(Writable... writables) { final DataOutputBuffer out = new DataOutputBuffer(); try { for(Writable w : writables) { w.write(out); } out.close(); } catch (IOException e) { throw new RuntimeException("Fail to convert writables to a byte array",e); } return out.getData(); }
/** * Creates a ByteBuffer with serialized {@link Credentials}. * * @param creds The credentials. * @return The ByteBuffer with the credentials. * @throws IOException */ public static ByteBuffer createTokenBuffer(Credentials creds) throws IOException { DataOutputBuffer dob = new DataOutputBuffer(); creds.writeTokenStorageToStream(dob); return ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); } }
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(numBuckets, routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
/** * Get the bytes for the token identifier * @return the bytes of the identifier */ public byte[] getBytes() { DataOutputBuffer buf = new DataOutputBuffer(4096); try { this.write(buf); } catch (IOException ie) { throw new RuntimeException("i/o error in getBytes", ie); } return Arrays.copyOf(buf.getData(), buf.getLength()); }
/** Workhorse to check and write out compressed data/lengths */ private synchronized void writeBuffer(DataOutputBuffer uncompressedDataBuffer) throws IOException { deflateFilter.resetState(); buffer.reset(); deflateOut.write(uncompressedDataBuffer.getData(), 0, uncompressedDataBuffer.getLength()); deflateOut.flush(); deflateFilter.finish(); WritableUtils.writeVInt(out, buffer.getLength()); out.write(buffer.getData(), 0, buffer.getLength()); }
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(numBuckets, routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
/** * Generate a string with the url-quoted base64 encoded serialized form * of the Writable. * @param obj the object to serialize * @return the encoded string * @throws IOException */ private static String encodeWritable(Writable obj) throws IOException { DataOutputBuffer buf = new DataOutputBuffer(); obj.write(buf); Base64 encoder = new Base64(0, null, true); byte[] raw = new byte[buf.getLength()]; System.arraycopy(buf.getData(), 0, raw, 0, buf.getLength()); return encoder.encodeToString(raw); }
/** Convert a string to a UTF-8 encoded byte array. * @see String#getBytes(String) */ public static byte[] getBytes(String string) { byte[] result = new byte[utf8Length(string)]; try { // avoid sync'd allocations DataOutputBuffer obuf = OBUF_FACTORY.get(); obuf.reset(); writeChars(obuf, string, 0, string.length()); System.arraycopy(obuf.getData(), 0, result, 0, obuf.getLength()); } catch (IOException e) { throw new RuntimeException(e); } return result; }
@Override public byte[] serialize() throws IOException { DataOutputBuffer dob = new DataOutputBuffer(5); dob.writeInt(masterKeyId); dob.write(index); byte[] b = dob.getData(); dob.close(); return b; }
private ByteBuffer serializeCredentials(Credentials credentials) throws IOException { Credentials containerCredentials = new Credentials(); containerCredentials.addAll(credentials); DataOutputBuffer containerTokens_dob = new DataOutputBuffer(); containerCredentials.writeTokenStorageToStream(containerTokens_dob); return ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength()); }
@Override protected boolean lessThan(Object a, Object b) { // indicate we're making progress if (progress != null) { progress.progress(); } SegmentDescriptor msa = (SegmentDescriptor)a; SegmentDescriptor msb = (SegmentDescriptor)b; return comparator.compare(msa.getKey().getData(), 0, msa.getKey().getLength(), msb.getKey().getData(), 0, msb.getKey().getLength()) < 0; } @Override
int numBuckets = edgeProp.getNumBuckets(); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, vertexType); DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); VertexManagerPluginDescriptor desc = VertexManagerPluginDescriptor.create( CustomPartitionVertex.class.getName()); byte[] userPayloadBytes = dob.getData(); ByteBuffer userPayload = ByteBuffer.wrap(userPayloadBytes); desc.setUserPayload(UserPayload.create(userPayload));
public static ByteBuffer serializeCredentials(Credentials credentials) throws IOException { Credentials containerCredentials = new Credentials(); containerCredentials.addAll(credentials); DataOutputBuffer containerTokensDob = new DataOutputBuffer(); containerCredentials.writeTokenStorageToStream(containerTokensDob); return ByteBuffer.wrap(containerTokensDob.getData(), 0, containerTokensDob.getLength()); } }
private void checkKey(WritableComparable key) throws IOException { // check that keys are well-ordered if (size != 0 && comparator.compare(lastKey, key) > 0) throw new IOException("key out of order: "+key+" after "+lastKey); // update lastKey with a copy of key by writing and reading outBuf.reset(); key.write(outBuf); // write new key inBuf.reset(outBuf.getData(), outBuf.getLength()); lastKey.readFields(inBuf); // read into lastKey }