Refine search
private <T, O> O roundTrip(Schema schema, T data, Class<? extends GenericData> modelClass) throws IOException { Job job = new Job(); AvroJob.setMapOutputKeySchema(job, schema); if (modelClass != null) AvroJob.setDataModelClass(job, modelClass); AvroSerialization serialization = ReflectionUtils.newInstance(AvroSerialization.class, job.getConfiguration()); Serializer<AvroKey<T>> serializer = serialization.getSerializer(AvroKey.class); Deserializer<AvroKey<O>> deserializer = serialization.getDeserializer(AvroKey.class); ByteArrayOutputStream baos = new ByteArrayOutputStream(); serializer.open(baos); serializer.serialize(new AvroKey<>(data)); serializer.close(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); deserializer.open(bais); AvroKey<O> result = null; result = deserializer.deserialize(result); deserializer.close(); return result.datum(); }
/** * Make a copy of the writable object using serialization to a buffer * @param src the object to copy from * @param dst the object to copy into, which is destroyed * @return dst param (the copy) * @throws IOException */ @SuppressWarnings("unchecked") public static <T> T copy(Configuration conf, T src, T dst) throws IOException { CopyInCopyOutBuffer buffer = CLONE_BUFFERS.get(); buffer.outBuffer.reset(); SerializationFactory factory = getFactory(conf); Class<T> cls = (Class<T>) src.getClass(); Serializer<T> serializer = factory.getSerializer(cls); serializer.open(buffer.outBuffer); serializer.serialize(src); buffer.moveData(); Deserializer<T> deserializer = factory.getDeserializer(cls); deserializer.open(buffer.inBuffer); dst = deserializer.deserialize(dst); return dst; }
Serialization serialization = new SerializationFactory(jobConf()).getSerialization(cls); deserializer.open(in); Object res = deserializer.deserialize(null); deserializer.close();
/** * Return a new instance of the given class with the deserialized data from * the input stream. */ public <T> T deser(Class clazz, InputStream in) throws IOException { Map<Class, Deserializer> deserializers = cachedDeserializers.get(); Deserializer deSer = deserializers.get(clazz); if(deSer == null) { deSer = serialization.getDeserializer(clazz); deserializers.put(clazz, deSer); } deSer.open(in); Object obj = deSer.deserialize(null); deSer.close(); return (T) obj; }
public DefaultStringifier(Configuration conf, Class<T> c) { SerializationFactory factory = new SerializationFactory(conf); this.serializer = factory.getSerializer(c); this.deserializer = factory.getDeserializer(c); this.inBuf = new DataInputBuffer(); this.outBuf = new DataOutputBuffer(); try { serializer.open(outBuf); deserializer.open(inBuf); } catch (IOException ex) { throw new RuntimeException(ex); } }
/** * Read the next value * @throws IOException */ private void readNextValue() throws IOException { DataInputBuffer nextValueBytes = in.getValue(); valueIn.reset(nextValueBytes.getData(), nextValueBytes.getPosition(), nextValueBytes.getLength()); value = valDeserializer.deserialize(value); } }
private <K> K serDeser(K conf) throws Exception { SerializationFactory factory = new SerializationFactory(CONF); Serializer<K> serializer = factory.getSerializer(GenericsUtil.getClass(conf)); Deserializer<K> deserializer = factory.getDeserializer(GenericsUtil.getClass(conf)); DataOutputBuffer out = new DataOutputBuffer(); serializer.open(out); serializer.serialize(conf); serializer.close(); DataInputBuffer in = new DataInputBuffer(); in.reset(out.getData(), out.getLength()); deserializer.open(in); K after = deserializer.deserialize(null); deserializer.close(); return after; }
Class<? extends CompressionCodec> codecClass = conf.getClassByName(codecClassname).asSubclass(CompressionCodec.class); this.codec = ReflectionUtils.newInstance(codecClass, conf); } catch (ClassNotFoundException cnfe) { throw new IllegalArgumentException("Unknown codec: " + valBuffer = new DataInputBuffer(); if (decompress) { valDecompressor = CodecPool.getDecompressor(codec); keyLenBuffer = new DataInputBuffer(); keyBuffer = new DataInputBuffer(); valLenBuffer = new DataInputBuffer(); new SerializationFactory(conf); this.keyDeserializer = getDeserializer(serializationFactory, getKeyClass()); this.keyDeserializer.open(valBuffer); } else { this.keyDeserializer.open(keyIn); + "custom serialization."); this.valDeserializer.open(valIn);
@Test @SuppressWarnings({"rawtypes", "unchecked"}) public void testWritableComparatorJavaSerialization() throws Exception { Serialization ser = new JavaSerialization(); Serializer<TestWC> serializer = ser.getSerializer(TestWC.class); DataOutputBuffer dob = new DataOutputBuffer(); serializer.open(dob); TestWC orig = new TestWC(0); serializer.serialize(orig); serializer.close(); Deserializer<TestWC> deserializer = ser.getDeserializer(TestWC.class); DataInputBuffer dib = new DataInputBuffer(); dib.reset(dob.getData(), 0, dob.getLength()); deserializer.open(dib); TestWC deser = deserializer.deserialize(null); deserializer.close(); assertEquals(orig, deser); }
/** * Deserialize an object using Hadoop serialization from a byte array. The * object cannot be null. */ public <T> T deser(Object obj, byte[] array, int offset, int length) throws IOException { Map<Class, Deserializer> deserializers = cachedDeserializers.get(); Deserializer deSer = deserializers.get(obj.getClass()); if(deSer == null) { deSer = serialization.getDeserializer(obj.getClass()); deserializers.put(obj.getClass(), deSer); } DataInputBuffer baIs = cachedInputStream.get(); baIs.reset(array, offset, length); deSer.open(baIs); obj = deSer.deserialize(obj); deSer.close(); baIs.close(); return (T) obj; } }
@Override public T fromString(String str) throws IOException { try { byte[] bytes = Base64.decodeBase64(str.getBytes("UTF-8")); inBuf.reset(bytes, bytes.length); T restored = deserializer.deserialize(null); return restored; } catch (UnsupportedCharsetException ex) { throw new IOException(ex.toString()); } }
/** {@inheritDoc} */ @Override public T decode(InputStream inStream) throws CoderException, IOException { Deserializer<T> deserializer = serialization.getDeserializer(type); deserializer.open(new UncloseableInputStream(inStream)); try { return deserializer.deserialize(null); } finally { deserializer.close(); } }
protected DeserializerComparator(Deserializer<T> deserializer) throws IOException { this.deserializer = deserializer; this.deserializer.open(buffer); }
public ValuesIterator (TezRawKeyValueIterator in, RawComparator<KEY> comparator, Class<KEY> keyClass, Class<VALUE> valClass, Configuration conf, TezCounter inputKeyCounter, TezCounter inputValueCounter) throws IOException { this.in = in; this.comparator = comparator; this.inputKeyCounter = inputKeyCounter; this.inputValueCounter = inputValueCounter; SerializationFactory serializationFactory = new SerializationFactory(conf); this.keyDeserializer = serializationFactory.getDeserializer(keyClass); this.keyDeserializer.open(keyIn); this.valDeserializer = serializationFactory.getDeserializer(valClass); this.valDeserializer.open(this.valueIn); }
@SuppressWarnings("unchecked") private Object deserializeValue(Object val) throws IOException { return valDeserializer.deserialize(val); }
/** {@inheritDoc} */ @Override public void close() throws IgniteCheckedException { try { serializer.close(); deserializer.close(); } catch (IOException e) { throw new IgniteCheckedException(e); } } }
/** * Return a new instance of the given class with the deserialized data from * the input stream. */ public <T> T deser(Class clazz, InputStream in) throws IOException { Map<Class, Deserializer> deserializers = cachedDeserializers.get(); Deserializer deSer = deserializers.get(clazz); if(deSer == null) { deSer = serialization.getDeserializer(clazz); deserializers.put(clazz, deSer); } deSer.open(in); Object obj = deSer.deserialize(null); deSer.close(); return (T) obj; }
public DefaultStringifier(Configuration conf, Class<T> c) { SerializationFactory factory = new SerializationFactory(conf); this.serializer = factory.getSerializer(c); this.deserializer = factory.getDeserializer(c); this.inBuf = new DataInputBuffer(); this.outBuf = new DataOutputBuffer(); try { serializer.open(outBuf); deserializer.open(inBuf); } catch (IOException ex) { throw new RuntimeException(ex); } }
/** * Read the next value * @throws IOException */ private void readNextValue() throws IOException { DataInputBuffer nextValueBytes = in.getValue(); valueIn.reset(nextValueBytes.getData(), nextValueBytes.getPosition(), nextValueBytes.getLength()); value = valDeserializer.deserialize(value); } }