public @Override void found(ObjectId object, @Nullable Integer storageSizeBytes) { super.found(object, storageSizeBytes); notifyProgressListener(); }
public @Override void inserted(ObjectId object, @Nullable Integer storageSizeBytes) { super.inserted(object, storageSizeBytes); notifyProgressListener(); }
private void testGetFeatures(final Iterable<Node> nodes, final int expectedSize) { CountingListener listener = BulkOpListener.newCountingListener(); Stopwatch sw = Stopwatch.createStarted(); Iterator<NodeRef> refs = Iterators.transform(nodes.iterator(), (n) -> NodeRef.create("layer", n)); Iterator<ObjectInfo<RevFeature>> iterator = db.getObjects(refs, listener, RevFeature.class); final int returnedObjectCount = Iterators.size(iterator); sw.stop(); System.err.printf("----- %,d Features queried (%,d not found) with getObjects() in %,dms\n", listener.found(), listener.notFound(), sw.elapsed(TimeUnit.MILLISECONDS)); System.err.printf("----- %s\n", sharedCache); Assert.assertEquals(listener.toString(), expectedSize, listener.found()); assertEquals(expectedSize, returnedObjectCount); }
treeInfos = db.getObjects(treeNodes.iterator(), listener, RevTree.class); assertEquals(numSubTrees, Iterators.size(treeInfos)); assertEquals(numSubTrees, listener.found()); assertEquals(0, listener.notFound()); featureInfos = db.getObjects(treeNodes.iterator(), listener, RevFeature.class); assertEquals(0, Iterators.size(featureInfos)); assertEquals(0, listener.found()); assertEquals(numSubTrees, listener.notFound()); featureInfos = db.getObjects(featureNodes.iterator(), listener, RevFeature.class); assertEquals(totalFeatures, Iterators.size(featureInfos)); assertEquals(totalFeatures, listener.found()); assertEquals(0, listener.notFound()); treeInfos = db.getObjects(featureNodes.iterator(), listener, RevTree.class); assertEquals(0, Iterators.size(treeInfos)); assertEquals(0, listener.found()); assertEquals(totalFeatures, listener.notFound());
treeInfos = db.getObjects(treeNodes.iterator(), listener, RevTree.class); assertEquals(numSubTrees, Iterators.size(treeInfos)); assertEquals(numSubTrees, listener.found()); assertEquals(0, listener.notFound()); featureInfos = db.getObjects(treeNodes.iterator(), listener, RevFeature.class); assertEquals(0, Iterators.size(featureInfos)); assertEquals(0, listener.found()); assertEquals(numSubTrees, listener.notFound()); featureInfos = db.getObjects(featureNodes.iterator(), listener, RevFeature.class); assertEquals(totalFeatures, Iterators.size(featureInfos)); assertEquals(totalFeatures, listener.found()); assertEquals(0, listener.notFound()); treeInfos = db.getObjects(featureNodes.iterator(), listener, RevTree.class); assertEquals(0, Iterators.size(treeInfos)); assertEquals(0, listener.found()); assertEquals(totalFeatures, listener.notFound());
RevTree.class); IndexInfo indexInfo = indexDef.getIndex(); CountingListener c = new BulkOpListener.CountingListener(); indexdb.putAll(missingContents, c);
BulkOpListener listener = BulkOpListener.composite(countingListener1, countingListener2); assertEquals(0, countingListener1.deleted()); assertEquals(0, countingListener1.inserted()); assertEquals(0, countingListener1.found()); assertEquals(0, countingListener1.notFound()); assertEquals(0, countingListener2.deleted()); assertEquals(0, countingListener2.inserted()); assertEquals(0, countingListener2.found()); assertEquals(0, countingListener2.notFound()); listener.notFound(ObjectId.NULL); assertEquals(2, countingListener1.found()); assertEquals(3, countingListener1.deleted()); assertEquals(4, countingListener1.inserted()); assertEquals(5, countingListener1.notFound()); assertEquals(2, countingListener2.found()); assertEquals(3, countingListener2.deleted()); assertEquals(4, countingListener2.inserted()); assertEquals(5, countingListener2.notFound());
assertEquals(2, listener.found()); assertEquals(3, listener.notFound()); assertEquals(Sets.newHashSet(f1.getId(), f3.getId()), Sets.newHashSet(Iterables.transform(features, (f) -> f.getId()))); assertEquals(3, listener.found()); assertEquals(2, listener.notFound()); assertEquals(Sets.newHashSet(t1.getId(), t2.getId(), t3.getId()), Sets.newHashSet(Iterables.transform(trees, (t) -> t.getId()))); assertEquals(5, listener.found()); assertEquals(0, listener.notFound()); assertEquals(Sets.newHashSet(f1.getId(), f3.getId(), t1.getId(), t2.getId(), t3.getId()), Sets.newHashSet(Iterables.transform(all, (o) -> o.getId())));
assertEquals(2, listener.found()); assertEquals(3, listener.notFound()); assertEquals(Sets.newHashSet(f1.getId(), f3.getId()), Sets.newHashSet(Iterables.transform(features, (f) -> f.getId()))); assertEquals(3, listener.found()); assertEquals(2, listener.notFound()); assertEquals(Sets.newHashSet(t1.getId(), t2.getId(), t3.getId()), Sets.newHashSet(Iterables.transform(trees, (t) -> t.getId()))); assertEquals(5, listener.found()); assertEquals(0, listener.notFound()); assertEquals(Sets.newHashSet(f1.getId(), f3.getId(), t1.getId(), t2.getId(), t3.getId()), Sets.newHashSet(Iterables.transform(all, (o) -> o.getId())));
@Test public void testForwardingListener() { CountingListener countingListener = BulkOpListener.newCountingListener(); ForwardingListener listener = new ForwardingListener(countingListener); assertEquals(0, countingListener.deleted()); assertEquals(0, countingListener.inserted()); assertEquals(0, countingListener.found()); assertEquals(0, countingListener.notFound()); listener.found(ObjectId.NULL, 0); listener.found(ObjectId.NULL, 0); listener.deleted(ObjectId.NULL); listener.deleted(ObjectId.NULL); listener.deleted(ObjectId.NULL); listener.inserted(ObjectId.NULL, 0); listener.inserted(ObjectId.NULL, 0); listener.inserted(ObjectId.NULL, 0); listener.inserted(ObjectId.NULL, 0); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); assertEquals(2, countingListener.found()); assertEquals(3, countingListener.deleted()); assertEquals(4, countingListener.inserted()); assertEquals(5, countingListener.notFound()); }
@Test public void testCountingListener() { CountingListener listener = BulkOpListener.newCountingListener(); assertEquals(0, listener.deleted()); assertEquals(0, listener.inserted()); assertEquals(0, listener.found()); assertEquals(0, listener.notFound()); listener.found(ObjectId.NULL, 0); listener.found(ObjectId.NULL, 0); listener.deleted(ObjectId.NULL); listener.deleted(ObjectId.NULL); listener.deleted(ObjectId.NULL); listener.inserted(ObjectId.NULL, 0); listener.inserted(ObjectId.NULL, 0); listener.inserted(ObjectId.NULL, 0); listener.inserted(ObjectId.NULL, 0); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); listener.notFound(ObjectId.NULL); assertEquals(2, listener.found()); assertEquals(3, listener.deleted()); assertEquals(4, listener.inserted()); assertEquals(5, listener.notFound()); }
/** * @return the number of objects parsed from the input stream */ public IngestResults ingest(final InputStream in, final Callback callback) { Iterator<RevObject> objects = streamToObjects(in); BulkOpListener listener = new BulkOpListener() { @Override public void inserted(final ObjectId objectId, @Nullable Integer storageSizeBytes) { callback.callback(new Supplier<RevObject>() { @Override public RevObject get() { return database.get(objectId); } }); } }; CountingListener countingListener = BulkOpListener.newCountingListener(); listener = BulkOpListener.composite(countingListener, listener); database.putAll(objects, listener); return new IngestResults(countingListener.inserted(), countingListener.found()); }
@Test public void testGetAllWithListener() { ImmutableList<RevObject> expected = ImmutableList.of(feature(0, null, "some value"), feature(1, "value", new Integer(111)), feature(2, (Object) null), RevTree.EMPTY); for (RevObject o : expected) { assertTrue(db.put(o)); } assertTrue(db.put(feature(5, "not queried 1"))); assertTrue(db.put(feature(6, "not queried 2"))); Function<RevObject, ObjectId> toId = p -> p.getId(); Iterable<ObjectId> ids = Iterables.transform(expected, toId); CountingListener listener = BulkOpListener.newCountingListener(); Iterable<ObjectId> notFound = ImmutableList.of(RevObjectTestSupport.hashString("notfound1"), RevObjectTestSupport.hashString("notfound2")); Iterator<RevObject> result = db.getAll(Iterables.concat(notFound, ids), listener); List<RevObject> actual = ImmutableList.copyOf(result); assertEquals(Sets.newHashSet(ids), Sets.newHashSet(Iterables.transform(actual, toId))); assertEquals(expected.size(), listener.found()); assertEquals(2, listener.notFound()); }
@Test public void testGetAllWithListener() { ImmutableList<RevObject> expected = ImmutableList.of(feature(0, null, "some value"), feature(1, "value", new Integer(111)), feature(2, (Object) null), RevTree.EMPTY); for (RevObject o : expected) { assertTrue(db.put(o)); } assertTrue(db.put(feature(5, "not queried 1"))); assertTrue(db.put(feature(6, "not queried 2"))); Function<RevObject, ObjectId> toId = p -> p.getId(); Iterable<ObjectId> ids = Iterables.transform(expected, toId); CountingListener listener = BulkOpListener.newCountingListener(); Iterable<ObjectId> notFound = ImmutableList.of(RevObjectTestSupport.hashString("notfound1"), RevObjectTestSupport.hashString("notfound2")); Iterator<RevObject> result = db.getAll(Iterables.concat(notFound, ids), listener); List<RevObject> actual = ImmutableList.copyOf(result); assertEquals(Sets.newHashSet(ids), Sets.newHashSet(Iterables.transform(actual, toId))); assertEquals(expected.size(), listener.found()); assertEquals(2, listener.notFound()); }
/** * Read in the changes from the provided input stream and call the provided callback for each * change. The input stream represents the output of another {@code BinaryPackedChanges} * instance. * * @param in the stream to read from * @param callback the callback to call for each item */ public void ingest(final InputStream in, Callback callback) { PacketReadingIterator readingIterator = new PacketReadingIterator(in); Iterator<RevObject> asObjects = asObjects(readingIterator, callback); ObjectStore objectDatabase = repository.objectDatabase(); CountingListener listener = BulkOpListener.newCountingListener(); objectDatabase.putAll(asObjects, listener); LOGGER.info("Ingested %,d objects. Inserted: %,d. Already existing: %,d\n", listener.inserted() + listener.found(), listener.inserted(), listener.found()); this.filtered = readingIterator.isFiltered(); }
@Test public void testDeleteAllWithListener() { ImmutableList<RevObject> objs = ImmutableList.of(feature(0, null, "some value"), feature(1, "value", new Integer(111)), feature(2, (Object) null)); for (RevObject o : objs) { assertTrue(db.put(o)); } ObjectId notInDb1 = RevObjectTestSupport.hashString("fake1"); ObjectId notInDb2 = RevObjectTestSupport.hashString("fake2"); Function<RevObject, ObjectId> toId = p -> p.getId(); Iterator<ObjectId> ids = concat(singletonIterator(notInDb1), transform(objs.iterator(), toId), singletonIterator(notInDb2)); CountingListener listener = BulkOpListener.newCountingListener(); db.deleteAll(ids, listener); assertEquals(3, listener.deleted()); assertEquals(2, listener.notFound()); }
@Test public void testDeleteAllWithListener() { ImmutableList<RevObject> objs = ImmutableList.of(feature(0, null, "some value"), feature(1, "value", new Integer(111)), feature(2, (Object) null)); for (RevObject o : objs) { assertTrue(db.put(o)); } ObjectId notInDb1 = RevObjectTestSupport.hashString("fake1"); ObjectId notInDb2 = RevObjectTestSupport.hashString("fake2"); Function<RevObject, ObjectId> toId = p -> p.getId(); Iterator<ObjectId> ids = concat(singletonIterator(notInDb1), transform(objs.iterator(), toId), singletonIterator(notInDb2)); CountingListener listener = BulkOpListener.newCountingListener(); db.deleteAll(ids, listener); assertEquals(3, listener.deleted()); assertEquals(2, listener.notFound()); }
public @Override void inserted(ObjectId object, @Nullable Integer storageSizeBytes) { super.inserted(object, storageSizeBytes); notifyProgressListener(); }
private void testGetAll(final Iterable<ObjectId> ids, final int expectedSize) { CountingListener getAllListener = BulkOpListener.newCountingListener(); Stopwatch sw = Stopwatch.createStarted(); final int returnedObjectCount = Iterators .size(db.getAll(ids, getAllListener, RevFeature.class)); sw.stop(); System.err.printf( "----- %,d random objects queried (%,d not found) with getAll() in %,dms\n", getAllListener.found(), getAllListener.notFound(), sw.elapsed(TimeUnit.MILLISECONDS)); System.err.printf("----- %s\n", sharedCache); Assert.assertEquals(getAllListener.toString(), expectedSize, getAllListener.found()); assertEquals(expectedSize, returnedObjectCount); }
public @Override String toString() { // return String.format( // "inserted %,d/%,d: commits: %,d, trees: %,d, buckets: %,d, features: %,d, ftypes: %,d", // super.inserted(), total.get(), commits.get(), trees.get(), buckets.get(), // features.get(), featureTypes.get()); return String.format( "inserted %,d/%,d: commits: %,d, trees: %,d, features: %,d, ftypes: %,d", super.inserted(), total.get(), commits.get(), trees.get(), features.get(), featureTypes.get()); }