@Override public void init(DictionaryInfo info, int baseId, String hdfsDir) throws IOException { this.baseId = baseId; this.builder = new TrieDictionaryBuilder(new StringBytesConverter()); }
public NumberDictionary build(int baseId) { byte[] trieBytes = buildTrieBytes(baseId); NumberDictionary2 r = new NumberDictionary2(trieBytes); return r; }
public void addValue(T value) { if (value == null) return; byte[] valueBytes = bytesConverter.convertToBytes(value); addValue(valueBytes); }
private void assertSameBehaviorAsTrie(TrieDictionaryForest<String> dict, ArrayList<String> strs, int baseId) { TrieDictionaryBuilder<String> trieBuilder = new TrieDictionaryBuilder<>(new StringBytesConverter()); for (String s : strs) { if (s != null) trieBuilder.addValue(s); } TrieDictionary<String> trie = trieBuilder.build(baseId); assertEquals(trie.getMaxId(), dict.getMaxId()); assertEquals(trie.getMinId(), dict.getMinId()); assertEquals(trie.getSize(), dict.getSize()); assertEquals(trie.getSizeOfId(), dict.getSizeOfId()); assertEquals(trie.getSizeOfValue(), dict.getSizeOfValue()); }
public Dictionary<String> getDictionary(String resourcePath) throws IOException { DictionaryInfo dictInfo = getDictionaryInfo(resourcePath); return dictInfo == null ? null : dictInfo.getDictionaryObject(); }
@SuppressWarnings("rawtypes") private static Dictionary strsToDict(Collection<String> strs) { TrieDictionaryBuilder<String> builder = new TrieDictionaryBuilder<>(new StringBytesConverter()); for (String str : strs) { builder.addValue(str); } return builder.build(0); }
private static TrieDictionaryBuilder<String> newDictBuilder(Iterable<String> str) { TrieDictionaryBuilder<String> b = new TrieDictionaryBuilder<String>(new StringBytesConverter()); for (String s : str) b.addValue(s); return b; }
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId, int treeSize) { TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); b.setMaxTrieTreeSize(treeSize); for (String s : strs) { b.addValue(s); } return b; }
public static TrieDictionaryForestBuilder<String> newDictBuilder(Iterable<String> strs, int baseId) { TrieDictionaryForestBuilder<String> b = new TrieDictionaryForestBuilder<String>(new StringBytesConverter(), baseId); for (String s : strs) b.addValue(s); return b; }
@Override public void init(DictionaryInfo info, int baseId, String hdfsDir) throws IOException { builder = new TrieDictionaryForestBuilder(new StringBytesConverter(), baseId); }
public void addValue(T value) { addValue(bytesConverter.convertToBytes(value)); }
private static DictionaryInfo createDictInfo(String[] values) { MockDictionary mockDict = new MockDictionary(); mockDict.values = values; DictionaryInfo info = new DictionaryInfo(); info.setDictionaryObject(mockDict); return info; }
public TrieDictionaryForestBuilder(BytesConverter<T> bytesConverter, int baseId, int maxTrieTreeSizeMB) { this.bytesConverter = bytesConverter; this.trieBuilder = new TrieDictionaryBuilder<T>(bytesConverter); this.baseId = baseId; this.curOffset = 0; this.maxTrieTreeSize = maxTrieTreeSizeMB * 1024 * 1024; }
@Override public void init(DictionaryInfo info, int baseId, String hdfsDir) throws IOException { this.baseId = baseId; this.builder = new NumberDictionaryBuilder(); }
private void traversePostOrderR(Node node, Visitor visitor, int level) { for (Node c : node.children) traversePostOrderR(c, visitor, level + 1); visitor.visit(node, level); }
public static void main(String[] args) throws Exception { int count = (int) (Integer.MAX_VALUE * 0.8 / 64); benchmarkStringDictionary(new RandomStrings(count)); }
private void initForestCache() { enableCache(); for (TrieDictionary<T> tree : trees) { //disable duplicate cache tree.disableCache(); } }
/** * out print trie for debug */ public void print() { print(System.out); }
public String pop() throws IOException { String result = this.cache; reload(); return result; }