private void loadNormal(MapJoinPersistableTableContainer container, ObjectInputStream in, Writable keyContainer, Writable valueContainer) throws Exception { int numKeys = in.readInt(); for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) { MapJoinKeyObject key = new MapJoinKeyObject(); key.read(keyContext, in, keyContainer); if (container.get(key) == null) { container.put(key, new MapJoinEagerRowContainer()); } MapJoinEagerRowContainer values = (MapJoinEagerRowContainer) container.get(key); values.read(valueContext, in, valueContainer); container.put(key, values); } }
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { AbstractSerDe serde = context.getSerDe(); ObjectInspector valueObjectInspector = context.getStandardOI(); long numRows = rowCount(); long numRowsWritten = 0L; out.writeLong(numRows); for (List<Object> row = first(); row != null; row = next()) { serde.serialize(row.toArray(), valueObjectInspector).write(out); ++numRowsWritten; } if(numRows != rowCount()) { throw new ConcurrentModificationException("Values was modified while persisting"); } if(numRowsWritten != numRows) { throw new IllegalStateException("Expected to write " + numRows + " but wrote " + numRowsWritten); } }
@Override public MapJoinRowContainer copy() { MapJoinEagerRowContainer result = new MapJoinEagerRowContainer(); for(List<Object> item : list) { result.addRow(item); } return result; }
@Before public void setup() throws Exception { rowContainer = new MapJoinEagerRowContainer(); rowContainer.addRow(VALUE); container = new HashMapWrapper(); } @Test
public void read(MapJoinObjectSerDeContext context, ObjectInputStream in, Writable container) throws IOException, SerDeException { long numRows = in.readLong(); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { container.readFields(in); read(context, container); } }
console = new LogHelper(LOG, isSilent); memoryExhaustionChecker = MemoryExhaustionCheckerFactory.getChecker(console, hconf, conf); emptyRowContainer.addRow(emptyObjectArray);
@Override public MapJoinRowContainer copy() { MapJoinEagerRowContainer result = new MapJoinEagerRowContainer(); for(List<Object> item : list) { result.addRow(item); } return result; }
@Test public void testSerialization() throws Exception { MapJoinRowContainer container1 = new MapJoinEagerRowContainer(); container1.addRow(new Object[]{ new Text("f0"), null, new ShortWritable((short)0xf)}); container1.addRow(Arrays.asList(new Object[]{ null, new Text("f1"), new ShortWritable((short)0xf)})); container1.addRow(new Object[]{ null, null, new ShortWritable((short)0xf)}); container1.addRow(Arrays.asList(new Object[]{ new Text("f0"), new Text("f1"), new ShortWritable((short)0x1)})); MapJoinRowContainer container2 = Utilities.serde(container1, "f0,f1,filter", "string,string,smallint"); Utilities.testEquality(container1, container2); Assert.assertEquals(4, container1.rowCount()); Assert.assertEquals(1, container2.getAliasFilter()); }
public void read(MapJoinObjectSerDeContext context, ObjectInputStream in, Writable container) throws IOException, SerDeException { long numRows = in.readLong(); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { container.readFields(in); read(context, container); } }
console = new LogHelper(LOG, isSilent); memoryExhaustionHandler = new MapJoinMemoryExhaustionHandler(console, conf.getHashtableMemoryUsage()); emptyRowContainer.addRow(emptyObjectArray);
private void loadNormal(MapJoinPersistableTableContainer container, ObjectInputStream in, Writable keyContainer, Writable valueContainer) throws Exception { int numKeys = in.readInt(); for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) { MapJoinKeyObject key = new MapJoinKeyObject(); key.read(keyContext, in, keyContainer); if (container.get(key) == null) { container.put(key, new MapJoinEagerRowContainer()); } MapJoinEagerRowContainer values = (MapJoinEagerRowContainer) container.get(key); values.read(valueContext, in, valueContainer); container.put(key, values); } }
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { AbstractSerDe serde = context.getSerDe(); ObjectInspector valueObjectInspector = context.getStandardOI(); long numRows = rowCount(); long numRowsWritten = 0L; out.writeLong(numRows); for (List<Object> row = first(); row != null; row = next()) { serde.serialize(row.toArray(), valueObjectInspector).write(out); ++numRowsWritten; } if(numRows != rowCount()) { throw new ConcurrentModificationException("Values was modified while persisting"); } if(numRowsWritten != numRows) { throw new IllegalStateException("Expected to write " + numRows + " but wrote " + numRowsWritten); } }
@Override public MapJoinRowContainer copy() { MapJoinEagerRowContainer result = new MapJoinEagerRowContainer(); for(List<Object> item : list) { result.addRow(item); } return result; }
if (rowContainer == null) { if(value.length != 0) { rowContainer = new MapJoinEagerRowContainer(); rowContainer.addRow(value); } else {
public void read(MapJoinObjectSerDeContext context, ObjectInputStream in, Writable container) throws IOException, SerDeException { long numRows = in.readLong(); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { container.readFields(in); read(context, container); } }
console = new LogHelper(LOG, isSilent); memoryExhaustionHandler = new MapJoinMemoryExhaustionHandler(console, conf.getHashtableMemoryUsage()); emptyRowContainer.addRow(emptyObjectArray);
static MapJoinEagerRowContainer serde( MapJoinRowContainer container, String columns, String types) throws Exception { MapJoinEagerRowContainer result = new MapJoinEagerRowContainer(); ByteArrayInputStream bais; ObjectInputStream in; ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(baos); LazyBinarySerDe serde = new LazyBinarySerDe(); Properties props = new Properties(); props.put(serdeConstants.LIST_COLUMNS, columns); props.put(serdeConstants.LIST_COLUMN_TYPES, types); SerDeUtils.initializeSerDe(serde, null, props, null); MapJoinObjectSerDeContext context = new MapJoinObjectSerDeContext(serde, true); container.write(context, out); out.close(); bais = new ByteArrayInputStream(baos.toByteArray()); in = new ObjectInputStream(bais); result.read(context, in, new BytesWritable()); return result; } }
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { SerDe serde = context.getSerDe(); ObjectInspector valueObjectInspector = context.getStandardOI(); long numRows = rowCount(); long numRowsWritten = 0L; out.writeLong(numRows); for (List<Object> row = first(); row != null; row = next()) { serde.serialize(row.toArray(), valueObjectInspector).write(out); ++numRowsWritten; } if(numRows != rowCount()) { throw new ConcurrentModificationException("Values was modifified while persisting"); } if(numRowsWritten != numRows) { throw new IllegalStateException("Expected to write " + numRows + " but wrote " + numRowsWritten); } }