static MapJoinEagerRowContainer serde( MapJoinRowContainer container, String columns, String types) throws Exception { MapJoinEagerRowContainer result = new MapJoinEagerRowContainer(); ByteArrayInputStream bais; ObjectInputStream in; ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(baos); LazyBinarySerDe serde = new LazyBinarySerDe(); Properties props = new Properties(); props.put(serdeConstants.LIST_COLUMNS, columns); props.put(serdeConstants.LIST_COLUMN_TYPES, types); SerDeUtils.initializeSerDe(serde, null, props, null); MapJoinObjectSerDeContext context = new MapJoinObjectSerDeContext(serde, true); container.write(context, out); out.close(); bais = new ByteArrayInputStream(baos.toByteArray()); in = new ObjectInputStream(bais); result.read(context, in, new BytesWritable()); return result; } }
@Override public void setSerde(MapJoinObjectSerDeContext keyContext, MapJoinObjectSerDeContext valueContext) throws SerDeException { keySerde = keyContext.getSerDe(); AbstractSerDe valSerde = valueContext.getSerDe(); if (writeHelper == null) { LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName()); if (keySerde instanceof BinarySortableSerDe && valSerde instanceof LazyBinarySerDe) { LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector(); writeHelper = new LazyBinaryKvWriter(keySerde, valSoi, valueContext.hasFilterTag()); internalValueOi = valSoi; sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders(); nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers(); notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers(); } else { writeHelper = new KeyValueWriter(keySerde, valSerde, valueContext.hasFilterTag()); internalValueOi = createInternalOi(valueContext); sortableSortOrders = null; nullMarkers = null; notNullMarkers = null; } } }
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { AbstractSerDe serde = context.getSerDe(); ObjectInspector objectInspector = context.getStandardOI(); Writable container = serde.serialize(key, objectInspector); container.write(out); }
@Explain(displayName = "keyContext", explainLevels = { Level.DEBUG }) public String getDebugKeyContext() { MapJoinObjectSerDeContext keyContext; try { AbstractSerDe keySerde = (AbstractSerDe) ReflectionUtil.newInstance( keyTblDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerde, null, keyTblDesc.getProperties(), null); keyContext = new MapJoinObjectSerDeContext(keySerde, false); } catch (SerDeException e) { return null; } return keyContext.stringify(); }
SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos, filterMaps)); valueContextStringList.add(pos + ":" + valueContext.stringify());
keySerde = keyCtx.getSerDe(); AbstractSerDe valSerde = valCtx.getSerDe(); (LazyBinaryStructObjectInspector) valSerde.getObjectInspector(); writeHelper = new MapJoinBytesTableContainer.LazyBinaryKvWriter(keySerde, valSoi, valCtx.hasFilterTag()); if (internalValueOi == null) { internalValueOi = valSoi;
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { AbstractSerDe serde = context.getSerDe(); ObjectInspector objectInspector = context.getStandardOI(); Writable container = serde.serialize(key, objectInspector); container.write(out); }
private LazyBinaryStructObjectInspector createInternalOi( MapJoinObjectSerDeContext valCtx) throws SerDeException { // We are going to use LBSerDe to serialize values; create OI for retrieval. List<? extends StructField> fields = ((StructObjectInspector) valCtx.getSerDe().getObjectInspector()).getAllStructFieldRefs(); List<String> colNames = new ArrayList<String>(fields.size()); List<ObjectInspector> colOis = new ArrayList<ObjectInspector>(fields.size()); for (int i = 0; i < fields.size(); ++i) { StructField field = fields.get(i); colNames.add(field.getFieldName()); // It would be nice if OI could return typeInfo... TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( field.getFieldObjectInspector().getTypeName()); colOis.add(LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(typeInfo)); } return LazyBinaryObjectInspectorFactory .getLazyBinaryStructObjectInspector(colNames, colOis); }
static MapJoinKeyObject serde(MapJoinKeyObject key, String columns, String types) throws Exception { MapJoinKeyObject result = new MapJoinKeyObject(); ByteArrayInputStream bais; ObjectInputStream in; ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream out = new ObjectOutputStream(baos); LazyBinarySerDe serde = new LazyBinarySerDe(); Properties props = new Properties(); props.put(serdeConstants.LIST_COLUMNS, columns); props.put(serdeConstants.LIST_COLUMN_TYPES, types); SerDeUtils.initializeSerDe(serde, null, props, null); MapJoinObjectSerDeContext context = new MapJoinObjectSerDeContext(serde, false); key.write(context, out); out.close(); bais = new ByteArrayInputStream(baos.toByteArray()); in = new ObjectInputStream(bais); result.read(context, in, new BytesWritable()); return result; }
@Override public void setSerde(MapJoinObjectSerDeContext keyCtx, MapJoinObjectSerDeContext valCtx) throws SerDeException { AbstractSerDe keySerde = keyCtx.getSerDe(), valSerde = valCtx.getSerDe(); if (writeHelper == null) { LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName()); // We assume this hashtable is loaded only when tez is enabled LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector(); writeHelper = new MapJoinBytesTableContainer.LazyBinaryKvWriter(keySerde, valSoi, valCtx.hasFilterTag()); if (internalValueOi == null) { internalValueOi = valSoi; } if (sortableSortOrders == null) { sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders(); } if (nullMarkers == null) { nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers(); } if (notNullMarkers == null) { notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers(); } } } }
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { AbstractSerDe serde = context.getSerDe(); ObjectInspector valueObjectInspector = context.getStandardOI(); long numRows = rowCount(); long numRowsWritten = 0L; out.writeLong(numRows); for (List<Object> row = first(); row != null; row = next()) { serde.serialize(row.toArray(), valueObjectInspector).write(out); ++numRowsWritten; } if(numRows != rowCount()) { throw new ConcurrentModificationException("Values was modified while persisting"); } if(numRowsWritten != numRows) { throw new IllegalStateException("Expected to write " + numRows + " but wrote " + numRowsWritten); } }
private LazyBinaryStructObjectInspector createInternalOi( MapJoinObjectSerDeContext valCtx) throws SerDeException { // We are going to use LBSerDe to serialize values; create OI for retrieval. List<? extends StructField> fields = ((StructObjectInspector) valCtx.getSerDe().getObjectInspector()).getAllStructFieldRefs(); List<String> colNames = new ArrayList<String>(fields.size()); List<ObjectInspector> colOis = new ArrayList<ObjectInspector>(fields.size()); for (int i = 0; i < fields.size(); ++i) { StructField field = fields.get(i); colNames.add(field.getFieldName()); // It would be nice if OI could return typeInfo... TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString( field.getFieldObjectInspector().getTypeName()); colOis.add(LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(typeInfo)); } return LazyBinaryObjectInspectorFactory .getLazyBinaryStructObjectInspector(colNames, colOis); }
@Before public void setup() throws Exception { key = new MapJoinKeyObject(KEY); rowContainer = new MapJoinEagerRowContainer(); rowContainer.addRow(VALUE); baos = new ByteArrayOutputStream(); out = new ObjectOutputStream(baos); LazyBinarySerDe keySerde = new LazyBinarySerDe(); Properties keyProps = new Properties(); keyProps.put(serdeConstants.LIST_COLUMNS, "v1"); keyProps.put(serdeConstants.LIST_COLUMN_TYPES, "string"); SerDeUtils.initializeSerDe(keySerde, null, keyProps, null); LazyBinarySerDe valueSerde = new LazyBinarySerDe(); Properties valueProps = new Properties(); valueProps.put(serdeConstants.LIST_COLUMNS, "v1"); valueProps.put(serdeConstants.LIST_COLUMN_TYPES, "string"); SerDeUtils.initializeSerDe(valueSerde, null, keyProps, null); containerSerde = new MapJoinTableContainerSerDe( new MapJoinObjectSerDeContext(keySerde, false), new MapJoinObjectSerDeContext(valueSerde, false)); container = new HashMapWrapper(); }
@Override public void setSerde(MapJoinObjectSerDeContext keyContext, MapJoinObjectSerDeContext valueContext) throws SerDeException { AbstractSerDe keySerde = keyContext.getSerDe(), valSerde = valueContext.getSerDe(); if (writeHelper == null) { LOG.info("Initializing container with " + keySerde.getClass().getName() + " and " + valSerde.getClass().getName()); if (keySerde instanceof BinarySortableSerDe && valSerde instanceof LazyBinarySerDe) { LazyBinaryStructObjectInspector valSoi = (LazyBinaryStructObjectInspector) valSerde.getObjectInspector(); writeHelper = new LazyBinaryKvWriter(keySerde, valSoi, valueContext.hasFilterTag()); internalValueOi = valSoi; sortableSortOrders = ((BinarySortableSerDe) keySerde).getSortOrders(); nullMarkers = ((BinarySortableSerDe) keySerde).getNullMarkers(); notNullMarkers = ((BinarySortableSerDe) keySerde).getNotNullMarkers(); } else { writeHelper = new KeyValueWriter(keySerde, valSerde, valueContext.hasFilterTag()); internalValueOi = createInternalOi(valueContext); sortableSortOrders = null; nullMarkers = null; notNullMarkers = null; } } }
@Override public void write(MapJoinObjectSerDeContext context, ObjectOutputStream out) throws IOException, SerDeException { AbstractSerDe serde = context.getSerDe(); ObjectInspector valueObjectInspector = context.getStandardOI(); long numRows = rowCount(); long numRowsWritten = 0L; out.writeLong(numRows); for (List<Object> row = first(); row != null; row = next()) { serde.serialize(row.toArray(), valueObjectInspector).write(out); ++numRowsWritten; } if(numRows != rowCount()) { throw new ConcurrentModificationException("Values was modified while persisting"); } if(numRowsWritten != numRows) { throw new IllegalStateException("Expected to write " + numRows + " but wrote " + numRowsWritten); } }
@SuppressWarnings("deprecation") public static MapJoinKey read(Output output, MapJoinObjectSerDeContext context, Writable writable) throws SerDeException, HiveException { AbstractSerDe serde = context.getSerDe(); Object obj = serde.deserialize(writable); MapJoinKeyObject result = new MapJoinKeyObject(); result.read(serde.getObjectInspector(), obj); return result; }
public void generateMapMetaData() throws HiveException { // generate the meta data for key // index for key is -1 try { TableDesc keyTableDesc = conf.getKeyTblDesc(); AbstractSerDe keySerializer = (AbstractSerDe) ReflectionUtil.newInstance( keyTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(keySerializer, null, keyTableDesc.getProperties(), null); MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerializer, false); for (int pos = 0; pos < order.length; pos++) { if (pos == posBigTable) { continue; } TableDesc valueTableDesc; if (conf.getNoOuterJoin()) { valueTableDesc = conf.getValueTblDescs().get(pos); } else { valueTableDesc = conf.getValueFilteredTblDescs().get(pos); } AbstractSerDe valueSerDe = (AbstractSerDe) ReflectionUtil.newInstance( valueTableDesc.getDeserializerClass(), null); SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null); MapJoinObjectSerDeContext valueContext = new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos)); mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, valueContext); } } catch (SerDeException e) { throw new HiveException(e); } }
/** Called when loading the hashtable. */ public void add(MapJoinObjectSerDeContext context, BytesWritable value) throws HiveException { AbstractSerDe serde = context.getSerDe(); isAliasFilterSet = !context.hasFilterTag(); // has tag => need to set later if (rowLength == UNKNOWN) { try { rowLength = ObjectInspectorUtils.getStructSize(serde.getObjectInspector()); } catch (SerDeException ex) { throw new HiveException("Get structure size error", ex); } if (rowLength == 0) { array = EMPTY_OBJECT_ARRAY; } } if (rowLength > 0) { int rowCount = (array.length / rowLength); listRealloc(array.length + rowLength); read(serde, value, rowCount); } else { --rowLength; // see rowLength javadoc } }