kvContainer.clear(); spilledMapJoinTables[pos] = new MapJoinBytesTableContainer(restoredHashMap); spilledMapJoinTables[pos].setInternalValueOi(container.getInternalValueOi()); spilledMapJoinTables[pos].setSortableSortOrders(container.getSortableSortOrders()); spilledMapJoinTables[pos].setNullMarkers(container.getNullMarkers()); spilledMapJoinTables[pos].setNotNullMarkers(container.getNotNullMarkers());
@Override public void setSerde(MapJoinObjectSerDeContext keyContext, MapJoinObjectSerDeContext valueContext) throws SerDeException { keySerde = keyContext.getSerDe(); AbstractSerDe valSerde = valueContext.getSerDe(); if (writeHelper == null) { LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName()); if (keySerde instanceof BinarySortableSerDe && valSerde instanceof LazyBinarySerDe) { LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector(); writeHelper = new LazyBinaryKvWriter(keySerde, valSoi, valueContext.hasFilterTag()); internalValueOi = valSoi; sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders(); nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers(); notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers(); } else { writeHelper = new KeyValueWriter(keySerde, valSerde, valueContext.hasFilterTag()); internalValueOi = createInternalOi(valueContext); sortableSortOrders = null; nullMarkers = null; notNullMarkers = null; } } }
public ReusableRowContainer() { if (internalValueOi != null) { valueStruct = (LazyBinaryStruct) LazyBinaryFactory.createLazyBinaryObject(internalValueOi); needsComplexObjectFixup = MapJoinBytesTableContainer.hasComplexObjects(internalValueOi); if (needsComplexObjectFixup) { complexObjectArrayBuffer = new ArrayList<Object>( Collections.nCopies(internalValueOi.getAllStructFieldRefs().size(), null)); } else { complexObjectArrayBuffer = null; } } else { valueStruct = null; // No rows? needsComplexObjectFixup = false; complexObjectArrayBuffer = null; } uselessIndirection = new ByteArrayRef(); hashMapResult = new BytesBytesMultiHashMap.Result(); clearRows(); }
kvContainer.clear(); spilledMapJoinTables[pos] = new MapJoinBytesTableContainer(restoredHashMap); spilledMapJoinTables[pos].setInternalValueOi(container.getInternalValueOi()); spilledMapJoinTables[pos].setSortableSortOrders(container.getSortableSortOrders());
if (useOptimizedTables) { ObjectInspector keyOi = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOi)) { if (isFirstKey) { useOptimizedTables = false; if (useOptimizedTables) { if (!useHybridGraceHashJoin || isCrossProduct) { tableContainer = new MapJoinBytesTableContainer(hconf, valCtx, keyCount, 0); } else { tableContainer = new HybridHashTableContainer(hconf, keyCount, memory,
private static MapJoinTableContainer getDefaultEmptyContainer(Configuration hconf, MapJoinObjectSerDeContext keyCtx, MapJoinObjectSerDeContext valCtx) throws SerDeException { boolean useOptimizedContainer = HiveConf.getBoolVar( hconf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE); if (useOptimizedContainer) { return new MapJoinBytesTableContainer(hconf, valCtx, -1, 0); } MapJoinTableContainer container = new HashMapWrapper(); container.setSerde(keyCtx, valCtx); container.seal(); return container; } }
for (byte pos = 0; pos < order.length; pos++) { if (pos != alias && spilledMapJoinTables[pos] != null) { hashMapRowGetters[pos] = spilledMapJoinTables[pos].createGetter(refKey);
MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(); ObjectInspector keyOI = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOI)) { if (firstContainer) { LOG.warn("Not using optimized table container." +
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = MapJoinBytesTableContainer.getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
private void loadOptimized(MapJoinBytesTableContainer container, ObjectInputStream in, Writable key, Writable value) throws Exception { int numKeys = in.readInt(); for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) { key.readFields(in); long numRows = in.readLong(); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { value.readFields(in); container.putRow(key, value); } } }
if (useOptimizedTables) { ObjectInspector keyOi = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOi)) { if (isFirstKey) { useOptimizedTables = false; if (useOptimizedTables) { if (!useHybridGraceHashJoin || isCrossProduct) { tableContainer = new MapJoinBytesTableContainer(hconf, valCtx, keyCount, 0); } else { tableContainer = new HybridHashTableContainer(hconf, keyCount, memory,
if (tableContainer == null) { tableContainer = useOptimizedContainer ? new MapJoinBytesTableContainer(hconf, valueContext, -1, 0) : create(name, metaData);
for (byte pos = 0; pos < order.length; pos++) { if (pos != alias && spilledMapJoinTables[pos] != null) { hashMapRowGetters[pos] = spilledMapJoinTables[pos].createGetter(refKey);
MapJoinObjectSerDeContext keyCtx = mapJoinTableSerdes[pos].getKeyContext(); ObjectInspector keyOI = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOI)) { if (firstContainer) { LOG.warn("Not using optimized table container." +
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
private void loadOptimized(MapJoinBytesTableContainer container, ObjectInputStream in, Writable key, Writable value) throws Exception { int numKeys = in.readInt(); for (int keyIndex = 0; keyIndex < numKeys; keyIndex++) { key.readFields(in); long numRows = in.readLong(); for (long rowIndex = 0L; rowIndex < numRows; rowIndex++) { value.readFields(in); container.putRow(key, value); } } }
kvContainer.clear(); spilledMapJoinTables[pos] = new MapJoinBytesTableContainer(restoredHashMap); spilledMapJoinTables[pos].setInternalValueOi(container.getInternalValueOi()); spilledMapJoinTables[pos].setSortableSortOrders(container.getSortableSortOrders()); spilledMapJoinTables[pos].setNullMarkers(container.getNullMarkers()); spilledMapJoinTables[pos].setNotNullMarkers(container.getNotNullMarkers());
if (useOptimizedTables) { ObjectInspector keyOi = keyCtx.getSerDe().getObjectInspector(); if (!MapJoinBytesTableContainer.isSupportedKey(keyOi)) { if (isFirstKey) { useOptimizedTables = false; ? (useHybridGraceHashJoin ? new HybridHashTableContainer(hconf, keyCount, memory, desc.getParentDataSizes().get(pos), nwayConf) : new MapJoinBytesTableContainer(hconf, valCtx, keyCount, 0)) : new HashMapWrapper(hconf, keyCount); LOG.info("Using tableContainer " + tableContainer.getClass().getSimpleName());
if (tableContainer == null) { tableContainer = useOptimizedContainer ? new MapJoinBytesTableContainer(hconf, valueContext, -1, 0) : create(name, metaData);
@Override public void setSerde(MapJoinObjectSerDeContext keyContext, MapJoinObjectSerDeContext valueContext) throws SerDeException { AbstractSerDe keySerde = keyContext.getSerDe(), valSerde = valueContext.getSerDe(); if (writeHelper == null) { LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName()); if (keySerde instanceof BinarySortableSerDe && valSerde instanceof LazyBinarySerDe) { LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector(); writeHelper = new LazyBinaryKvWriter(keySerde, valSoi, valueContext.hasFilterTag()); internalValueOi = valSoi; sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders(); nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers(); notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers(); } else { writeHelper = new KeyValueWriter(keySerde, valSerde, valueContext.hasFilterTag()); internalValueOi = createInternalOi(valueContext); sortableSortOrders = null; nullMarkers = null; notNullMarkers = null; } } }