public void persist(ObjectOutputStream out, MapJoinPersistableTableContainer tableContainer) throws HiveException { int numKeys = tableContainer.size(); try { out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(numKeys); for(Map.Entry<MapJoinKey, MapJoinRowContainer> entry : tableContainer.entrySet()) { entry.getKey().write(keyContext, out); entry.getValue().write(valueContext, out); } } catch (SerDeException e) { String msg = "SerDe error while attempting to persist table container"; throw new HiveException(msg, e); } catch(IOException e) { String msg = "IO error while attempting to persist table container"; throw new HiveException(msg, e); } if(numKeys != tableContainer.size()) { throw new ConcurrentModificationException("TableContainer was modified while persisting: " + tableContainer); } }
MapJoinRowContainer rowContainer = tableContainer.get(key); if (rowContainer == null) { if(value.length != 0) { memoryExhaustionChecker.checkMemoryOverhead(rowNumber, hashTableScale, tableContainer.size()); tableContainer.put(key, rowContainer); } else if (rowContainer == emptyRowContainer) { rowContainer = rowContainer.copy(); rowContainer.addRow(value); tableContainer.put(key, rowContainer); } else { rowContainer.addRow(value);
public static void persistDummyTable(ObjectOutputStream out) throws IOException { MapJoinPersistableTableContainer tableContainer = new HashMapWrapper(); out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(tableContainer.size()); }
private void loadNormal(MapJoinPersistableTableContainer container, ObjectInputStream in, Writable keyContainer, Writable valueContainer) throws Exception { int numKeys = in.readInt(); for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) { MapJoinKeyObject key = new MapJoinKeyObject(); key.read(keyContext, in, keyContainer); if (container.get(key) == null) { container.put(key, new MapJoinEagerRowContainer()); } MapJoinEagerRowContainer values = (MapJoinEagerRowContainer) container.get(key); values.read(valueContext, in, valueContainer); container.put(key, values); } }
@Test public void testDummyContainer() throws Exception { MapJoinTableContainerSerDe.persistDummyTable(out); out.close(); in = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); container = containerSerde.load(in); Assert.assertEquals(0, container.size()); Assert.assertTrue(container.entrySet().isEmpty()); } }
MapJoinEagerRowContainer values = new MapJoinEagerRowContainer(); values.read(valueContext, in, valueContainer); tableContainer.put(key, values);
Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag + " with group count: " + tableContainer.size() + " into file: " + path); out.close(); tableContainer.clear(); FileStatus status = fs.getFileStatus(path); console.printInfo(Utilities.now() + "\tUploaded 1 File to: " + path +
private void loadNormal(MapJoinPersistableTableContainer container, ObjectInputStream in, Writable keyContainer, Writable valueContainer) throws Exception { int numKeys = in.readInt(); for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) { MapJoinKeyObject key = new MapJoinKeyObject(); key.read(keyContext, in, keyContainer); if (container.get(key) == null) { container.put(key, new MapJoinEagerRowContainer()); } MapJoinEagerRowContainer values = (MapJoinEagerRowContainer) container.get(key); values.read(valueContext, in, valueContainer); container.put(key, values); } }
public static void persistDummyTable(ObjectOutputStream out) throws IOException { MapJoinPersistableTableContainer tableContainer = new HashMapWrapper(); out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(tableContainer.size()); }
MapJoinEagerRowContainer values = new MapJoinEagerRowContainer(); values.read(valueContext, in, valueContainer); tableContainer.put(key, values);
MapJoinRowContainer rowContainer = tableContainer.get(key); if (rowContainer == null) { if(value.length != 0) { memoryExhaustionHandler.checkMemoryStatus(tableContainer.size(), rowNumber); tableContainer.put(key, rowContainer); } else if (rowContainer == emptyRowContainer) { rowContainer = rowContainer.copy(); rowContainer.addRow(value); tableContainer.put(key, rowContainer); } else { rowContainer.addRow(value);
public void persist(ObjectOutputStream out, MapJoinPersistableTableContainer tableContainer) throws HiveException { int numKeys = tableContainer.size(); try { out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(numKeys); for(Map.Entry<MapJoinKey, MapJoinRowContainer> entry : tableContainer.entrySet()) { entry.getKey().write(keyContext, out); entry.getValue().write(valueContext, out); } } catch (SerDeException e) { String msg = "SerDe error while attempting to persist table container"; throw new HiveException(msg, e); } catch(IOException e) { String msg = "IO error while attempting to persist table container"; throw new HiveException(msg, e); } if(numKeys != tableContainer.size()) { throw new ConcurrentModificationException("TableContainer was modified while persisting: " + tableContainer); } }
Path path = Utilities.generatePath(tmpURI, dumpFilePrefix, tag, fileName); console.printInfo(Utilities.now() + "\tDump the side-table for tag: " + tag + " with group count: " + tableContainer.size() + " into file: " + path); out.close(); tableContainer.clear(); FileStatus status = fs.getFileStatus(path); console.printInfo(Utilities.now() + "\tUploaded 1 File to: " + path +
@Test public void testSerialization() throws Exception { container.put(key, rowContainer); containerSerde.persist(out, container); out.close(); in = new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray())); container = containerSerde.load(in); Utilities.testEquality(rowContainer, container.get(key)); } @Test
public static void persistDummyTable(ObjectOutputStream out) throws IOException { MapJoinPersistableTableContainer tableContainer = new HashMapWrapper(); out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(tableContainer.size()); }
MapJoinEagerRowContainer values = new MapJoinEagerRowContainer(); values.read(valueContext, in, valueContainer); tableContainer.put(key, values);
MapJoinRowContainer rowContainer = tableContainer.get(key); if (rowContainer == null) { if(value.length != 0) { memoryExhaustionHandler.checkMemoryStatus(tableContainer.size(), rowNumber); tableContainer.put(key, rowContainer); } else if (rowContainer == emptyRowContainer) { rowContainer = rowContainer.copy(); rowContainer.addRow(value); tableContainer.put(key, rowContainer); } else { rowContainer.addRow(value);
public void persist(ObjectOutputStream out, MapJoinPersistableTableContainer tableContainer) throws HiveException { int numKeys = tableContainer.size(); try { out.writeUTF(tableContainer.getClass().getName()); out.writeObject(tableContainer.getMetaData()); out.writeInt(numKeys); for(Map.Entry<MapJoinKey, MapJoinRowContainer> entry : tableContainer.entrySet()) { entry.getKey().write(keyContext, out); entry.getValue().write(valueContext, out); } } catch (SerDeException e) { String msg = "SerDe error while attempting to persist table container"; throw new HiveException(msg, e); } catch(IOException e) { String msg = "IO error while attempting to persist table container"; throw new HiveException(msg, e); } if(numKeys != tableContainer.size()) { throw new ConcurrentModificationException("TableContainer was modified while persisting: " + tableContainer); } }