private CompiledNode compileNode(UnCompiledNode<T> nodeIn, int tailLength) throws IOException { final long node; long bytesPosStart = bytes.getPosition(); if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.numArcs <= 1) && tailLength <= shareMaxTailLength) { if (nodeIn.numArcs == 0) { node = fst.addNode(this, nodeIn); lastFrozenNode = node; } else { node = dedupHash.add(this, nodeIn); } } else { node = fst.addNode(this, nodeIn); } assert node != -2; long bytesPosEnd = bytes.getPosition(); if (bytesPosEnd != bytesPosStart) { // The FST added a new node: assert bytesPosEnd > bytesPosStart; lastFrozenNode = node; } nodeIn.clear(); final CompiledNode fn = new CompiledNode(); fn.node = node; return fn; }
public long add(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException { //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" + mask); final long h = hash(nodeIn); long pos = h & mask; int c = 0; while(true) { final long v = table.get(pos); if (v == 0) { // freeze & add final long node = fst.addNode(builder, nodeIn); //System.out.println(" now freeze node=" + node); assert hash(node) == h : "frozenHash=" + hash(node) + " vs h=" + h; count++; table.set(pos, node); // Rehash at 2/3 occupancy: if (count > 2*table.size()/3) { rehash(); } return node; } else if (nodesEqual(nodeIn, v)) { // same node is already here return v; } // quadratic probe pos = (pos + (++c)) & mask; } }
private CompiledNode compileNode(UnCompiledNode<T> nodeIn, int tailLength) throws IOException { final long node; long bytesPosStart = bytes.getPosition(); if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.numArcs <= 1) && tailLength <= shareMaxTailLength) { if (nodeIn.numArcs == 0) { node = fst.addNode(this, nodeIn); lastFrozenNode = node; } else { node = dedupHash.add(this, nodeIn); } } else { node = fst.addNode(this, nodeIn); } assert node != -2; long bytesPosEnd = bytes.getPosition(); if (bytesPosEnd != bytesPosStart) { // The FST added a new node: assert bytesPosEnd > bytesPosStart; lastFrozenNode = node; } nodeIn.clear(); final CompiledNode fn = new CompiledNode(); fn.node = node; return fn; }
private CompiledNode compileNode(UnCompiledNode<T> nodeIn, int tailLength) throws IOException { final long node; long bytesPosStart = bytes.getPosition(); if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.numArcs <= 1) && tailLength <= shareMaxTailLength) { if (nodeIn.numArcs == 0) { node = fst.addNode(this, nodeIn); lastFrozenNode = node; } else { node = dedupHash.add(this, nodeIn); } } else { node = fst.addNode(this, nodeIn); } assert node != -2; long bytesPosEnd = bytes.getPosition(); if (bytesPosEnd != bytesPosStart) { // The FST added a new node: assert bytesPosEnd > bytesPosStart; lastFrozenNode = node; } nodeIn.clear(); final CompiledNode fn = new CompiledNode(); fn.node = node; return fn; }
private CompiledNode compileNode(UnCompiledNode<T> nodeIn, int tailLength) throws IOException { final long node; long bytesPosStart = bytes.getPosition(); if (dedupHash != null && (doShareNonSingletonNodes || nodeIn.numArcs <= 1) && tailLength <= shareMaxTailLength) { if (nodeIn.numArcs == 0) { node = fst.addNode(this, nodeIn); lastFrozenNode = node; } else { node = dedupHash.add(this, nodeIn); } } else { node = fst.addNode(this, nodeIn); } assert node != -2; long bytesPosEnd = bytes.getPosition(); if (bytesPosEnd != bytesPosStart) { // The FST added a new node: assert bytesPosEnd > bytesPosStart; lastFrozenNode = node; } nodeIn.clear(); final CompiledNode fn = new CompiledNode(); fn.node = node; return fn; }
public long add(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException { //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" + mask); final long h = hash(nodeIn); long pos = h & mask; int c = 0; while(true) { final long v = table.get(pos); if (v == 0) { // freeze & add final long node = fst.addNode(builder, nodeIn); //System.out.println(" now freeze node=" + node); assert hash(node) == h : "frozenHash=" + hash(node) + " vs h=" + h; count++; table.set(pos, node); // Rehash at 2/3 occupancy: if (count > 2*table.size()/3) { rehash(); } return node; } else if (nodesEqual(nodeIn, v)) { // same node is already here return v; } // quadratic probe pos = (pos + (++c)) & mask; } }
public long add(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException { //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" + mask); final long h = hash(nodeIn); long pos = h & mask; int c = 0; while(true) { final long v = table.get(pos); if (v == 0) { // freeze & add final long node = fst.addNode(builder, nodeIn); //System.out.println(" now freeze node=" + node); assert hash(node) == h : "frozenHash=" + hash(node) + " vs h=" + h; count++; table.set(pos, node); // Rehash at 2/3 occupancy: if (count > 2*table.size()/3) { rehash(); } return node; } else if (nodesEqual(nodeIn, v)) { // same node is already here return v; } // quadratic probe pos = (pos + (++c)) & mask; } }
public long add(Builder<T> builder, Builder.UnCompiledNode<T> nodeIn) throws IOException { //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" + mask); final long h = hash(nodeIn); long pos = h & mask; int c = 0; while(true) { final long v = table.get(pos); if (v == 0) { // freeze & add final long node = fst.addNode(builder, nodeIn); //System.out.println(" now freeze node=" + node); assert hash(node) == h : "frozenHash=" + hash(node) + " vs h=" + h; count++; table.set(pos, node); // Rehash at 2/3 occupancy: if (count > 2*table.size()/3) { rehash(); } return node; } else if (nodesEqual(nodeIn, v)) { // same node is already here return v; } // quadratic probe pos = (pos + (++c)) & mask; } }