@Test public void testGetAllRecordsWithMeta() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); int number = 2; Set<String> added = newHashSet(); for (int i = 0; i < number; i++) { String rec = dataStore.addRecord(randomStream(i, 16516)) .getIdentifier().toString(); added.add(rec); } if (dataStore.getType() == SharedDataStore.Type.SHARED) { dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "1meta"); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "2meta"); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "ameta1"); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "bmeta2"); } for (int i = 0; i < number; i++) { String rec = dataStore.addRecord(randomStream(100+i, 16516)) .getIdentifier().toString(); added.add(rec); } Set<String> retrieved = newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()), new Function<DataRecord, String>() { @Nullable @Override public String apply(@Nullable DataRecord input) { return input.getIdentifier().toString(); } })); assertEquals(added, retrieved); }
String repoId1 = UUID.randomUUID().toString(); data.repoIds.add(repoId1); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); dataStore.addMetadataRecord(new FileInputStream(f), getName(extended, repoId1, suffix1)); String suffix2 = String.valueOf(System.currentTimeMillis()); data.suffixes.add(suffix2); dataStore.addMetadataRecord(f2, getName(true, repoId1, suffix2)); String suffix3 = String.valueOf(System.currentTimeMillis()); data.suffixes.add(suffix3); dataStore.addMetadataRecord(f3, getName(extended, repoId2, suffix3));
@Test public void gcMarkOnRemote() throws Exception { MemoryBlobStoreNodeStore memNodeStore = new MemoryBlobStoreNodeStore(setupDataStore); String rep2Id = ClusterRepositoryInfo.getOrCreateId(memNodeStore); setupDataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(rep2Id)); Map<String, String> idMapping = dummyData(memNodeStore, rep2Id, store, setupDataStore, temporaryFolder.newFile()); File dump = temporaryFolder.newFolder(); Data data = prepareData(storeFixture, blobFixture, 10, 5, 1); data.added.addAll(idMapping.keySet()); data.idToPath.putAll(idMapping); storeFixture.close(); testGc(dump, data, 0, false); }
@Before public void setup() throws Exception { setupDataStore = blobFixture.init(temporaryFolder); store = storeFixture.init(setupDataStore, temporaryFolder.newFolder()); additionalParams = "--ds-read-write"; String repoId = ClusterRepositoryInfo.getOrCreateId(store); setupDataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); }
@Test public void gcNoMarkOnRemote() throws Exception { MemoryBlobStoreNodeStore memNodeStore = new MemoryBlobStoreNodeStore(setupDataStore); String rep2Id = ClusterRepositoryInfo.getOrCreateId(memNodeStore); setupDataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(rep2Id)); File dump = temporaryFolder.newFolder(); Data data = prepareData(storeFixture, blobFixture, 10, 5, 1); storeFixture.close(); List<String> argsList = Lists .newArrayList("--collect-garbage", "--max-age", String.valueOf(0), "--" + getOption(blobFixture.getType()), blobFixture.getConfigPath(), storeFixture.getConnectionString(), "--out-dir", dump.getAbsolutePath(), "--reset-log-config", "false", "--work-dir", temporaryFolder.newFolder().getAbsolutePath()); if (!Strings.isNullOrEmpty(additionalParams)) { argsList.add(additionalParams); } testIncorrectParams(argsList, Lists.newArrayList("Not all repositories have marked references available : "), MarkSweepGarbageCollector.class); }
private static Map<String, String> dummyData(MemoryBlobStoreNodeStore memNodeStore, String rep2Id, NodeStore store, DataStoreBlobStore setupDataStore, File f) throws IOException, CommitFailedException, DataStoreException { List<String> list = Lists.newArrayList(); Map<String, String> idMapping = Maps.newHashMap(); NodeBuilder a = memNodeStore.getRoot().builder(); for (int i = 0; i < 2; i++) { Blob b = store.createBlob(randomStream(i+100, 18342)); Iterator<String> idIter = setupDataStore.resolveChunks(b.getContentIdentity()); while (idIter.hasNext()) { String id = idIter.next(); list.add(id); idMapping.put(id, "/d" + i); } a.child("d" + i).setProperty("x", b); } memNodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY); writeStrings(list.iterator(), f, false); sort(f); setupDataStore.addMetadataRecord(f, REFERENCES.getNameFromId(rep2Id)); return idMapping; }
private MarkSweepGarbageCollector getGarbageCollector() throws DataStoreException, IOException { DataStoreBlobStore blobStore = (DataStoreBlobStore) getNodeStoreComponent(BlobStore.class); FileStore fileStore = getNodeStoreComponent(FileStore.class); File fileStoreRoot = getNodeStoreComponent(FileStore.class.getName() + ":root"); if (null == garbageCollector) { String repoId = ClusterRepositoryInfo.getOrCreateId(getNodeStore()); blobStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); if (null == executor) { executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); } garbageCollector = new MarkSweepGarbageCollector( new SegmentBlobReferenceRetriever(fileStore), blobStore, executor, fileStoreRoot.getAbsolutePath(), 2048, 0, repoId ); } return garbageCollector; }
@Test public void testAddMetadata() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); String repoId = UUID.randomUUID().toString(); Set<String> refs = Sets.newHashSet("1_1", "1_2"); File f = folder.newFile(); FileIOUtils.writeStrings(refs.iterator(), f, false); dataStore.addMetadataRecord(new FileInputStream(f), SharedStoreRecordType.REFERENCES.getNameFromId(repoId)); assertTrue(dataStore.metadataRecordExists(SharedStoreRecordType.REFERENCES.getNameFromId(repoId))); DataRecord rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId)); Set<String> refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false); assertEquals(refs, refsReturned); assertEquals( SharedStoreRecordType.REFERENCES.getIdFromName(rec.getIdentifier().toString()), repoId); dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); }
dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); DataRecord repo1 = dataStore.getMetadataRecord(SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); DataRecord repo2 = dataStore.getMetadataRecord(SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1)); DataRecord markerRec1 = dataStore.getMetadataRecord(SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REFERENCES.getNameFromId(repoId1)); DataRecord rec1 = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId1)); long lastModifiedRec1 = rec1.getLastModified(); TimeUnit.MILLISECONDS.sleep(25); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REFERENCES.getNameFromId(repoId2)); DataRecord rec2 = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId2));