@Override public boolean addValue(String value) { if (value == null) return false; builder.addValue(value); return true; }
public void addValue(T value) { addValue(bytesConverter.convertToBytes(value)); }
private static Dictionary newDictionaryOfString() { TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter()); builder.addValue("Dong"); builder.addValue("George"); builder.addValue("Jason"); builder.addValue("Kejia"); builder.addValue("Luke"); builder.addValue("Mahone"); builder.addValue("Qianhao"); builder.addValue("Shaofeng"); builder.addValue("Xu"); builder.addValue("Yang"); return builder.build(0); }
public static void main(String[] args) throws Exception { TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); b.addValue("part"); b.print(); b.addValue("part"); b.print(); b.addValue("par"); b.print(); b.addValue("partition"); b.print(); b.addValue("party"); b.print(); b.addValue("parties"); b.print(); b.addValue("paint"); b.print(); TrieDictionary<String> dict = b.build(0);
private void addValue(byte[] valueBytes) { ByteArray valueByteArray = new ByteArray(valueBytes); if (previousValue != null && isOrdered) { int comp = previousValue.compareTo(valueByteArray); if (comp == 0) { return; //duplicate value } if (comp > 0) { logger.info("values not in ascending order, previous '{}', current '{}'", previousValue, valueByteArray); isOrdered = false; if (trees.size() > 0) { throw new IllegalStateException("Invalid input data. Unordered data cannot be split into multi trees"); } } } previousValue = valueByteArray; trieBuilder.addValue(valueBytes); curTreeSize += valueBytes.length; if (curTreeSize >= maxTrieTreeSize && isOrdered) { TrieDictionary<T> tree = trieBuilder.build(0); addTree(tree); reset(); } }
row[j] = null; b.addValue(row[j]);
private void checkOverflowParts(Node node) { LinkedList<Node> childrenCopy = new LinkedList<Node>(node.children); for (Node child : childrenCopy) { if (child.part.length > 255) { byte[] first255 = Arrays.copyOf(child.part, 255); completeParts.append(node.part); completeParts.append(first255); byte[] visited = completeParts.retrieve(); this.addValue(visited); completeParts.withdraw(255); completeParts.withdraw(node.part.length); } } completeParts.append(node.part); // by here the node.children may have been changed for (Node child : node.children) { checkOverflowParts(child); } completeParts.withdraw(node.part.length); }
@Test public void testSuperLongStringValue() { String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); String v1 = longPrefix + "xyz"; b.addValue(v1); String strLen200 = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghid"; b.addValue(strLen200); StringBuilder sb = new StringBuilder(); for (int i = 0; i < 25; i++) { sb.append(strLen200); } String strLen5000 = sb.toString(); b.addValue(strLen5000); TrieDictionary<String> dict = b.build(0); dict.dump(System.out); sb.setLength(0); for (int j = 0; j < 7; j++) { sb.append(strLen5000); } String strLen35000 = sb.toString(); b.addValue(strLen35000); Exception ex = null; try { b.build(0); } catch (Exception e) { ex = e; } Assert.assertNotNull(ex); }
private static TrieDictionaryBuilder<String> newDictBuilder(Iterable<String> str) { TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); for (String s : str) b.addValue(s); return b; }
TrieDictionaryBuilder<String> oldTrieBuilder = new TrieDictionaryBuilder<>(new StringBytesConverter()); for (String str : testData) oldTrieBuilder.addValue(str); TrieDictionary<String> oldDict = oldTrieBuilder.build(0); keep |= oldDict.getIdFromValue(testData.get(0));
String cell = row[column.getZeroBasedIndex()]; if (cell != null) b.addValue(cell);
@SuppressWarnings("rawtypes") private static Dictionary strsToDict(Collection<String> strs) { TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter()); for (String str : strs) { builder.addValue(str); } return builder.build(0); }
dictBuilder.addValue(str);
@Before public void before() { int dataSize = 100 * 10000; TrieDictionaryBuilder<String> b1 = new TrieDictionaryBuilder<>(new StringBytesConverter()); TrieDictionaryForestBuilder<String> b2 = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), 0, 5); this.rawData = genStringDataSet(dataSize); for (String str : this.rawData) { b1.addValue(str); b2.addValue(str); } this.oldDict = b1.build(0); this.newDict = b2.build(); System.out.println("new dict split tree size : " + ((TrieDictionaryForest<String>) newDict).getTrees().size()); }
public void addValue(T value) { addValue(bytesConverter.convertToBytes(value)); }
@Override public void addValue(byte[] value) { codec.encodeNumber(value, 0, value.length); byte[] copy = Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen); super.addValue(copy); }
private static Dictionary buildStringDict(List<byte[]> values, int baseId, int nSamples, ArrayList samples) { TrieDictionaryBuilder builder = new TrieDictionaryBuilder(new StringBytesConverter()); for (byte[] value : values) { String v = Bytes.toString(value); builder.addValue(v); if (samples.size() < nSamples && samples.contains(v) == false) samples.add(v); } return builder.build(baseId); }
private void assertSameBehaviorAsTrie(TrieDictionaryForest<String> dict, ArrayList<String> strs, int baseId) { TrieDictionaryBuilder<String> trieBuilder = new TrieDictionaryBuilder<>(new StringBytesConverter()); for (String s : strs) { if (s != null) trieBuilder.addValue(s); } TrieDictionary<String> trie = trieBuilder.build(baseId); assertEquals(trie.getMaxId(), dict.getMaxId()); assertEquals(trie.getMinId(), dict.getMinId()); assertEquals(trie.getSize(), dict.getSize()); assertEquals(trie.getSizeOfId(), dict.getSizeOfId()); assertEquals(trie.getSizeOfValue(), dict.getSizeOfValue()); }
private static TrieDictionaryBuilder<String> newDictBuilder(ArrayList<String> str) { TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); for (String s : str) b.addValue(s); return b; }
@Test public void testSuperLongStringValue() { String longPrefix = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" + "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"; TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); String v1 = longPrefix + "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"; String v2 = longPrefix + "xyz"; b.addValue(v1); b.addValue(v2); TrieDictionary<String> dict = b.build(0); dict.dump(System.out); }