public static Dictionary mergeDictionaries(DictionaryInfo targetInfo, List<DictionaryInfo> sourceDicts) { HashSet<byte[]> dedup = new HashSet<byte[]>(); for (DictionaryInfo info : sourceDicts) { Dictionary<?> dict = info.getDictionaryObject(); int minkey = dict.getMinId(); int maxkey = dict.getMaxId(); byte[] buffer = new byte[dict.getSizeOfValue()]; for (int i = minkey; i <= maxkey; ++i) { int size = dict.getValueBytesFromId(i, buffer, 0); dedup.add(Bytes.copy(buffer, 0, size)); } } List<byte[]> valueList = new ArrayList<byte[]>(); valueList.addAll(dedup); return buildDictionaryFromValueList(targetInfo, valueList); }
@Test public void testBitMapContainer() { // create container BitMapContainer container = new BitMapContainer(info.getDigest(), 0); Dictionary<String> dict = info.dict(0); for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) { container.append(v); } container.append(Dictionary.NULL_ID[dict.getSizeOfId()]); container.closeForChange(); // copy by serialization List<ImmutableBytesWritable> bytes = container.toBytes(); BitMapContainer container2 = new BitMapContainer(info.getDigest(), 0); container2.fromBytes(bytes); // check the copy int i = 0; for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) { int value = container2.getValueIntAt(i++); assertEquals(v, value); } assertEquals(Dictionary.NULL_ID[dict.getSizeOfId()], container2.getValueIntAt(i++)); assertEquals(container, container2); }
ImmutableBytesWritable bytes = new ImmutableBytesWritable(buf); for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) { BytesUtil.writeUnsigned(v, buf, 0, dict.getSizeOfId()); container.append(bytes); for (int v = dict.getMinId(); v <= dict.getMaxId(); v++) { container2.getValueAt(i++, bytes); int value = BytesUtil.readUnsigned(bytes.get(), bytes.getOffset(),