/** Read a compressed buffer */ private synchronized void readBuffer(DataInputBuffer buffer, CompressionInputStream filter) throws IOException { // Read data into a temporary buffer DataOutputBuffer dataBuffer = new DataOutputBuffer(); try { int dataBufferLength = WritableUtils.readVInt(in); dataBuffer.write(in, dataBufferLength); // Set up 'buffer' connected to the input-stream buffer.reset(dataBuffer.getData(), 0, dataBuffer.getLength()); } finally { dataBuffer.close(); } // Reset the codec filter.resetState(); }
/** Used by child copy constructors. */ protected synchronized void copy(Writable other) { if (other != null) { try { DataOutputBuffer out = new DataOutputBuffer(); other.write(out); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); readFields(in); } catch (IOException e) { throw new IllegalArgumentException("map cannot be copied: " + e.getMessage()); } } else { throw new IllegalArgumentException("source map cannot be null"); } }
/** * Parses all the RegionInfo instances from the passed in stream until EOF. Presumes the * RegionInfo's were serialized to the stream with oDelimitedByteArray() * @param bytes serialized bytes * @param offset the start offset into the byte[] buffer * @param length how far we should read into the byte[] buffer * @return All the RegionInfos that are in the byte array. Keeps reading till we hit the end. * @throws IOException */ static List<RegionInfo> parseDelimitedFrom(final byte[] bytes, final int offset, final int length) throws IOException { if (bytes == null) { throw new IllegalArgumentException("Can't build an object with empty bytes array"); } DataInputBuffer in = new DataInputBuffer(); List<RegionInfo> ris = new ArrayList<>(); try { in.reset(bytes, offset, length); while (in.available() > 0) { RegionInfo ri = parseFrom(in); ris.add(ri); } } finally { in.close(); } return ris; } }
/** * Convert a UTF-8 encoded byte array back into a string. * * @throws IOException if the byte array is invalid UTF8 */ public static String fromBytes(byte[] bytes) throws IOException { DataInputBuffer dbuf = new DataInputBuffer(); dbuf.reset(bytes, 0, bytes.length); StringBuilder buf = new StringBuilder(bytes.length); readChars(dbuf, buf, bytes.length); return buf.toString(); }
throw new IllegalArgumentException("Writable cannot be null"); DataInputBuffer in = new DataInputBuffer(); try { in.reset(bytes, offset, length); w.readFields(in); return w; } finally { in.close();
public static void testValue(int val, int vintlen) throws IOException { DataOutputBuffer buf = new DataOutputBuffer(); DataInputBuffer inbuf = new DataInputBuffer(); WritableUtils.writeVInt(buf, val); if (LOG.isDebugEnabled()) { LOG.debug("Value = " + val); BytesWritable printer = new BytesWritable(); printer.set(buf.getData(), 0, buf.getLength()); LOG.debug("Buffer = " + printer); } inbuf.reset(buf.getData(), 0, buf.getLength()); assertEquals(val, WritableUtils.readVInt(inbuf)); assertEquals(vintlen, buf.getLength()); assertEquals(vintlen, WritableUtils.getVIntSize(val)); assertEquals(vintlen, WritableUtils.decodeVIntSize(buf.getData()[0])); }
@Test public void testFSCheckpointIDSerialization() throws IOException { Path inpath = new Path("/tmp/blah"); FSCheckpointID cidin = new FSCheckpointID(inpath); DataOutputBuffer out = new DataOutputBuffer(); cidin.write(out); out.close(); FSCheckpointID cidout = new FSCheckpointID(null); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), 0, out.getLength()); cidout.readFields(in); in.close(); assert cidin.equals(cidout); }
if (key.getClass() != getKeyClass()) throw new IOException("wrong key class: "+key.getClass().getName() +" is not "+keyClass); outBuf.reset(); keyLength = next(outBuf); if (keyLength < 0) return false; valBuffer.reset(outBuf.getData(), outBuf.getLength()); valBuffer.mark(0); if (valBuffer.getPosition() != keyLength) throw new IOException(key + " read " + valBuffer.getPosition() + " bytes, should read " + keyLength); } else { readBlock(); } catch (EOFException eof) { return false; int keyLength = WritableUtils.readVInt(keyLenIn);
public TextRecordInputStream(FileStatus f) throws IOException { final Path fpath = f.getPath(); final Configuration lconf = getConf(); r = new SequenceFile.Reader(lconf, SequenceFile.Reader.file(fpath)); key = ReflectionUtils.newInstance( r.getKeyClass().asSubclass(Writable.class), lconf); val = ReflectionUtils.newInstance( r.getValueClass().asSubclass(Writable.class), lconf); inbuf = new DataInputBuffer(); outbuf = new DataOutputBuffer(); }
@Override public int read() throws IOException { int ret; if (null == inbuf || -1 == (ret = inbuf.read())) { if (!r.next(key, val)) { return -1; } byte[] tmp = key.toString().getBytes(StandardCharsets.UTF_8); outbuf.write(tmp, 0, tmp.length); outbuf.write('\t'); tmp = val.toString().getBytes(StandardCharsets.UTF_8); outbuf.write(tmp, 0, tmp.length); outbuf.write('\n'); inbuf.reset(outbuf.getData(), outbuf.getLength()); outbuf.reset(); ret = inbuf.read(); } return ret; }
/** * Read the next value * @throws IOException */ private void readNextValue() throws IOException { DataInputBuffer nextValueBytes = in.getValue(); valueIn.reset(nextValueBytes.getData(), nextValueBytes.getPosition(), nextValueBytes.getLength() - nextValueBytes.getPosition()); value = valDeserializer.deserialize(value); }
private void checkKey(WritableComparable key) throws IOException { // check that keys are well-ordered if (size != 0 && comparator.compare(lastKey, key) > 0) throw new IOException("key out of order: "+key+" after "+lastKey); // update lastKey with a copy of key by writing and reading outBuf.reset(); key.write(outBuf); // write new key inBuf.reset(outBuf.getData(), outBuf.getLength()); lastKey.readFields(inBuf); // read into lastKey }
public DefaultStringifier(Configuration conf, Class<T> c) { SerializationFactory factory = new SerializationFactory(conf); this.serializer = factory.getSerializer(c); this.deserializer = factory.getDeserializer(c); this.inBuf = new DataInputBuffer(); this.outBuf = new DataOutputBuffer(); try { serializer.open(outBuf); deserializer.open(inBuf); } catch (IOException ex) { throw new RuntimeException(ex); } }
public static int compare(DataInputBuffer buf1, DataOutputBuffer buf2) { byte[] b1 = buf1.getData(); byte[] b2 = buf2.getData(); int s1 = buf1.getPosition(); int s2 = 0; int l1 = buf1.getLength(); int l2 = buf2.getLength(); return FastByteComparisons.compareTo(b1, s1, (l1 - s1), b2, s2, l2); }
public void append(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); if (keyLength < 0) { throw new IOException("Negative key-length not allowed: " + keyLength + " for " + key); } int valueLength = value.getLength() - value.getPosition(); if (valueLength < 0) { throw new IOException("Negative value-length not allowed: " + valueLength + " for " + value); } WritableUtils.writeVInt(out, keyLength); WritableUtils.writeVInt(out, valueLength); out.write(key.getData(), key.getPosition(), keyLength); out.write(value.getData(), value.getPosition(), valueLength); }
public static TestSignable deserialize(byte[] bytes) throws IOException { DataInputBuffer db = new DataInputBuffer(); db.reset(bytes, bytes.length); int keyId = db.readInt(); byte b = db.readByte(); db.close(); return new TestSignable(keyId, b); }
@Override public T fromString(String str) throws IOException { try { byte[] bytes = Base64.decodeBase64(str.getBytes("UTF-8")); inBuf.reset(bytes, bytes.length); T restored = deserializer.deserialize(null); return restored; } catch (UnsupportedCharsetException ex) { throw new IOException(ex.toString()); } }
valInFilter.resetState(); valBuffer.reset(); } else { readBuffer(valLenBuffer, valLenInFilter); readBuffer(valBuffer, valInFilter); noBufferedValues = noBufferedRecords; valuesDecompressed = true; int currentKey = noBufferedKeys + 1; for (int i=noBufferedValues; i > currentKey; --i) { skipValBytes += WritableUtils.readVInt(valLenIn); --noBufferedValues;
/** * Streaming access to the key. Useful for desrializing the key into * user objects. * * @return The input stream. */ public DataInputStream getKeyStream() { keyDataInputStream.reset(keyBuffer, klen); return keyDataInputStream; }