@Test public void testSerialization() throws Exception { container.put(key, rowContainer); containerSerde.persist(out, container); out.close(); in = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); container = containerSerde.load(in); Utilities.testEquality(rowContainer, container.get(key)); } @Test
return getDefaultEmptyContainer(hconf, keyContext, valueContext); return getDefaultEmptyContainer(hconf, keyContext, valueContext); tableContainer = useOptimizedContainer ? new MapJoinBytesTableContainer(hconf, valueContext, -1, 0) : create(name, metaData); loadOptimized((MapJoinBytesTableContainer) tableContainer, in, keyContainer, valueContainer); } else { loadNormal((MapJoinPersistableTableContainer) tableContainer, in, keyContainer, valueContainer);
private MapJoinTableContainer loadMapJoinTableContainer( FileSystem fs, Path path, MapJoinTableContainerSerDe mapJoinTableSerde) throws HiveException { return useFastContainer ? mapJoinTableSerde.loadFastContainer(desc, fs, path, hconf) : mapJoinTableSerde.load(fs, path, hconf); }
@Test public void testDummyContainer() throws Exception { MapJoinTableContainerSerDe.persistDummyTable(out); out.close(); in = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); container = containerSerde.load(in); Assert.assertEquals(0, container.size()); Assert.assertTrue(container.entrySet().isEmpty()); } }
MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(), valCtx = mapJoinTableSerdes[pos].getValueContext(); if (useOptimizedTables) { ObjectInspector keyOi = keyCtx.getSerDe().getObjectInspector();
@Before public void setup() throws Exception { key = new MapJoinKeyObject(KEY); rowContainer = new MapJoinEagerRowContainer(); rowContainer.addRow(VALUE); baos = new ByteArrayOutputStream(); out = new ObjectOutputStream(baos); LazyBinarySerDe keySerde = new LazyBinarySerDe(); Properties keyProps = new Properties(); keyProps.put(serdeConstants.LIST_COLUMNS, "v1"); keyProps.put(serdeConstants.LIST_COLUMN_TYPES, "string"); SerDeUtils.initializeSerDe(keySerde, null, keyProps, null); LazyBinarySerDe valueSerde = new LazyBinarySerDe(); Properties valueProps = new Properties(); valueProps.put(serdeConstants.LIST_COLUMNS, "v1"); valueProps.put(serdeConstants.LIST_COLUMN_TYPES, "string"); SerDeUtils.initializeSerDe(valueSerde, null, keyProps, null); containerSerde = new MapJoinTableContainerSerDe( new MapJoinObjectSerDeContext(keySerde, false), new MapJoinObjectSerDeContext(valueSerde, false)); container = new HashMapWrapper(); }
os = fs.create(path, numReplication); out = new ObjectOutputStream(new BufferedOutputStream(os, 4096)); mapJoinTableSerde.persist(out, tableContainer); } finally { if (out != null) {
new FileInputStream(path.toUri().getPath()))); try{ mapJoinTables[pos] = mapJoinTableSerdes[pos].load(in); } finally { in.close();
String name = in.readUTF(); Map<String, String> metaData = (Map<String, String>) in.readObject(); tableContainer = create(name, metaData); } catch (IOException e) { throw new HiveException("IO error while trying to create table container", e);
MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(); ObjectInspector keyOI = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOI)) {
MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(), valCtx = mapJoinTableSerdes[pos].getValueContext(); if (useOptimizedTables) { ObjectInspector keyOi = keyCtx.getSerDe().getObjectInspector();
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
ObjectOutputStream out = new ObjectOutputStream(new BufferedOutputStream(fs.create(path))); try { mapJoinTableSerdes[tag].persist(out, tableContainer); } finally { out.close();
new FileInputStream(path.toUri().getPath()), 4096)); try{ mapJoinTables[pos] = mapJoinTableSerdes[pos].load(in); } finally { in.close();
String name = in.readUTF(); Map<String, String> metaData = (Map<String, String>) in.readObject(); tableContainer = create(name, metaData); } catch (IOException e) { throw new HiveException("IO error while trying to create table container", e);
MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(); ObjectInspector keyOI = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOI)) {
mapJoinTableContainerSerDe.getKeyContext(), mapJoinTableContainerSerDe.getValueContext()); break; case FAST:
return getDefaultEmptyContainer(keyContext, valueContext); return getDefaultEmptyContainer(keyContext, valueContext); tableContainer = useOptimizedContainer ? new MapJoinBytesTableContainer(hconf, valueContext, -1, 0) : create(name, metaData); loadOptimized((MapJoinBytesTableContainer) tableContainer, in, keyContainer, valueContainer); } else { loadNormal((MapJoinPersistableTableContainer) tableContainer, in, keyContainer, valueContainer);
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
os = fs.create(path, replication); out = new ObjectOutputStream(new BufferedOutputStream(os, 4096)); mapJoinTableSerde.persist(out, tableContainer); } finally { if (out != null) {