@Override public List<DataRecord> getAllMetadataRecords(String prefix) { if (delegate instanceof SharedDataStore) { return ((SharedDataStore) delegate).getAllMetadataRecords(prefix); } return null; }
@Override public List<DataRecord> getAllMetadataRecords(String prefix) { if (delegate instanceof SharedDataStore) { return ((SharedDataStore) delegate).getAllMetadataRecords(prefix); } return null; }
@Override public List<DataRecord> getAllMetadataRecords(String prefix) { if (delegate instanceof SharedDataStore) { return ((SharedDataStore) delegate).getAllMetadataRecords(prefix); } return null; }
LOG.trace("Retrieving all blob id files available form the DataStore"); Iterable<DataRecord> refRecords = datastore.getAllMetadataRecords(fileNamePrefix);
LOG.trace("Retrieving all blob id files available form the DataStore"); Iterable<DataRecord> refRecords = datastore.getAllMetadataRecords(fileNamePrefix);
LOG.trace("Retrieving all blob id files available form the DataStore"); Iterable<DataRecord> refRecords = datastore.getAllMetadataRecords(fileNamePrefix);
@Test public void testBackendDeleteAllMetadataRecordsNoRecordsNoChange() { SharedDataStore fds = dataStore; assertEquals(0, fds.getAllMetadataRecords("").size()); fds.deleteAllMetadataRecords(""); assertEquals(0, fds.getAllMetadataRecords("").size()); }
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords( SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified();
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords( SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified();
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords( SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified();
assertEquals(count, fds.getAllMetadataRecords(prefix).size()); assertEquals(0, fds.getAllMetadataRecords(prefix).size());
@Test public void testBackendGetAllMetadataRecordsPrefixMatchesAll() throws DataStoreException { SharedDataStore fds = dataStore; assertEquals(0, fds.getAllMetadataRecords("").size()); String prefixAll = "prefix1"; String prefixSome = "prefix1.prefix2"; String prefixOne = "prefix1.prefix3"; String prefixNone = "prefix4"; fds.addMetadataRecord(randomStream(1, 10), String.format("%s.testRecord1", prefixAll)); fds.addMetadataRecord(randomStream(2, 10), String.format("%s.testRecord2", prefixSome)); fds.addMetadataRecord(randomStream(3, 10), String.format("%s.testRecord3", prefixSome)); fds.addMetadataRecord(randomStream(4, 10), String.format("%s.testRecord4", prefixOne)); fds.addMetadataRecord(randomStream(5, 10), "prefix5.testRecord5"); assertEquals(5, fds.getAllMetadataRecords("").size()); assertEquals(4, fds.getAllMetadataRecords(prefixAll).size()); assertEquals(2, fds.getAllMetadataRecords(prefixSome).size()); assertEquals(1, fds.getAllMetadataRecords(prefixOne).size()); assertEquals(0, fds.getAllMetadataRecords(prefixNone).size()); fds.deleteAllMetadataRecords(""); assertEquals(0, fds.getAllMetadataRecords("").size()); }
@Test public void testBackendGetAllMetadataRecordsNullPrefixThrowsNullPointerException() { expectedEx.expect(IllegalArgumentException.class); expectedEx.expectMessage("prefix should not be null"); SharedDataStore fds = dataStore; fds.getAllMetadataRecords(null); }
@Test public void testBackendDeleteAllMetadataRecordsPrefixMatchesAll() throws DataStoreException { SharedDataStore fds = dataStore; String prefixAll = "prefix1"; String prefixSome = "prefix1.prefix2"; String prefixOne = "prefix1.prefix3"; String prefixNone = "prefix4"; Map<String, Integer> prefixCounts = Maps.newHashMap(); prefixCounts.put(prefixAll, 4); prefixCounts.put(prefixSome, 2); prefixCounts.put(prefixOne, 1); prefixCounts.put(prefixNone, 0); for (Map.Entry<String, Integer> entry : prefixCounts.entrySet()) { fds.addMetadataRecord(randomStream(1, 10), String.format("%s.testRecord1", prefixAll)); fds.addMetadataRecord(randomStream(2, 10), String.format("%s.testRecord2", prefixSome)); fds.addMetadataRecord(randomStream(3, 10), String.format("%s.testRecord3", prefixSome)); fds.addMetadataRecord(randomStream(4, 10), String.format("%s.testRecord4", prefixOne)); int preCount = fds.getAllMetadataRecords("").size(); fds.deleteAllMetadataRecords(entry.getKey()); int deletedCount = preCount - fds.getAllMetadataRecords("").size(); assertEquals(entry.getValue().intValue(), deletedCount); fds.deleteAllMetadataRecords(""); } }
@Test public void snapshotIgnoreAfterRemove() throws Exception { LOG.info("In snapshotIgnoreAfterRemove"); BlobIdTracker.SnapshotJob job = tracker.new SnapshotJob(); snapshotRemove(job, false); // Since already retrieved the datastore should be empty unless the snapshot has actually run ScheduledFuture<?> scheduledFuture = scheduler.schedule(job, 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); assertTrue("Snapshot not skipped", read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType())).isEmpty()); }
@Test public void snapshotBeforeRemove() throws Exception { LOG.info("In snapshotBeforeRemove"); Clock clock = Clock.ACCURATE; BlobIdTracker.SnapshotJob job = tracker.new SnapshotJob(100, clock); //Mimic an intervening snapshot between add and remove by skipping the retrieve call. Set<String> present = snapshotRemove(job, true); clock.waitUntil(System.currentTimeMillis() + 100); // Since already retrieved the datastore should not be empty unless the snapshot is ignored ScheduledFuture<?> scheduledFuture = scheduler.schedule(job, 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); assertEquals("Elements not equal after snapshot after remove", present, read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType()))); }
@Test public void snapshotExecuteAfterRemove() throws Exception { LOG.info("In snapshotExecuteAfterRemove"); Clock clock = Clock.ACCURATE; BlobIdTracker.SnapshotJob job = tracker.new SnapshotJob(100, clock); // Mimics a call to get after add and before remove similar to the calls in GC Set<String> present = snapshotRemove(job, false); clock.waitUntil(System.currentTimeMillis() + 100); // Since already retrieved the datastore should not be empty unless the snapshot is ignored ScheduledFuture<?> scheduledFuture = scheduler.schedule(job, 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); assertEquals("Elements not equal after snapshot after remove", present, read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType()))); }
@Test public void addSnapshot() throws Exception { LOG.info("In addSnapshot"); Set<String> initAdd = add(tracker, range(0, 4)); ScheduledFuture<?> scheduledFuture = scheduler.schedule(tracker.new SnapshotJob(), 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); Set<String> retrieved = retrieve(tracker); assertEquals("Extra elements after add", initAdd, retrieved); assertTrue(read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType())).isEmpty()); }
@Test public void externalAddOffline() throws Exception { LOG.info("In externalAddOffline"); // Close and open a new object to use the system property closer.close(); root = folder.newFolder(); File blobIdRoot = new File(root, "blobids"); blobIdRoot.mkdirs(); //Add file offline File offline = new File(blobIdRoot, "blob-offline123456.gen"); List<String> offlineLoad = range(0, 1000); FileIOUtils.writeStrings(offlineLoad.iterator(), offline, false); this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore); this.scheduler = newSingleThreadScheduledExecutor(); closer.register(tracker); closer.register(new ExecutorCloser(scheduler)); Set<String> initAdd = add(tracker, range(1001, 1005)); ScheduledFuture<?> scheduledFuture = scheduler.schedule(tracker.new SnapshotJob(), 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); initAdd.addAll(offlineLoad); assertEquals(initAdd.size(), Iterators.size(tracker.get())); Set<String> retrieved = retrieve(tracker); assertEquals("Extra elements after add", initAdd, retrieved); assertTrue(read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType())).isEmpty()); }
private Set<String> snapshotRemove(BlobIdTracker.SnapshotJob job, boolean skipGetBeforeRemove) throws Exception { Set<String> initAdd = add(tracker, range(0, 4)); ScheduledFuture<?> scheduledFuture = scheduler.schedule(job, 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); if (!skipGetBeforeRemove) { assertEquals("Extra elements after add", initAdd, retrieve(tracker)); } remove(tracker, folder.newFile(), initAdd, range(1, 2)); assertEquals("Extra elements after removes synced with datastore", initAdd, read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType()))); assertEquals("Extra elements after remove", initAdd, retrieve(tracker)); return initAdd; }