private DataStoreBlobStore createDataStore() { FileDataStore fds = new OakFileDataStore(); fds.setPath(dir.getAbsolutePath()); fds.setMinRecordLength(maxInlineSize); fds.init(null); return new DataStoreBlobStore(fds); }
private DataStoreBlobStore createDataStore() { FileDataStore fds = new OakFileDataStore(); fds.setPath(dir.getAbsolutePath()); fds.setMinRecordLength(maxInlineSize); fds.init(null); return new DataStoreBlobStore(fds); }
@Override public BlobStore setUp() { fds = new FileDataStore(); fds.setMinRecordLength(4092); storeDir = new File(basedir, unique); fds.init(storeDir.getAbsolutePath()); configure(fds); DataStoreBlobStore bs = new DataStoreBlobStore(fds, true, fdsCacheInMB); bs.setBlobStatsCollector(new BlobStoreStats(statisticsProvider)); configure(bs); return bs; }
private DataStoreBlobStore createDataStore() { FileDataStore fds = new OakFileDataStore(); fds.setPath(dir.getAbsolutePath()); fds.setMinRecordLength(maxInlineSize); fds.init(null); return new DataStoreBlobStore(fds); }
@Override public BlobStore setUp() { fds = new FileDataStore(); fds.setMinRecordLength(4092); storeDir = new File(basedir, unique); fds.init(storeDir.getAbsolutePath()); configure(fds); DataStoreBlobStore bs = new DataStoreBlobStore(fds, true, fdsCacheInMB); bs.setBlobStatsCollector(new BlobStoreStats(statisticsProvider)); configure(bs); return bs; }
@Override protected DataStore createDataStore() throws RepositoryException { FileDataStore fds = new FileDataStore(); Properties props = loadProperties("/fs.properties"); String pathValue = props.getProperty(FSBackend.FS_BACKEND_PATH); if (pathValue != null && !"".equals(pathValue.trim())) { fsPath = pathValue + "/fds" + "-" + String.valueOf(randomGen.nextInt(100000)) + "-" + String.valueOf(randomGen.nextInt(100000)); } else { fsPath = dataStoreDir + "/repository/datastore"; } LOG.info("path [{}] set.", fsPath); fds.setPath(fsPath); fds.init(dataStoreDir); return fds; }
private FileDataStore createFileDataStore() { FileDataStore fds = new FileDataStore(); fds.setMinRecordLength(4092); fds.init(getWorkDir().getAbsolutePath()); return fds; }
@Override protected void before() throws Throwable { FileDataStore fds = new FileDataStore(); configureDataStore(fds); fds.init(folder.newFolder().getAbsolutePath()); store = new DataStoreBlobStore(fds); }
private void testGetAllIdentifiers(String path, String unnormalizedPath) throws Exception { File testDir = new File(path); FileUtils.touch(new File(testDir, "ab/cd/ef/abcdef")); FileUtils.touch(new File(testDir, "bc/de/fg/bcdefg")); FileUtils.touch(new File(testDir, "cd/ef/gh/cdefgh")); FileUtils.touch(new File(testDir, "c")); FileDataStore fds = new OakFileDataStore(); fds.setPath(unnormalizedPath); fds.init(null); Iterator<DataIdentifier> dis = fds.getAllIdentifiers(); Set<String> fileNames = Sets.newHashSet(Iterators.transform(dis, new Function<DataIdentifier, String>() { @Override public String apply(@Nullable DataIdentifier input) { return input.toString(); } })); Set<String> expectedNames = Sets.newHashSet("abcdef","bcdefg","cdefgh"); assertEquals(expectedNames, fileNames); FileUtils.cleanDirectory(testDir); }
private void testGetAllIdentifiers(String path, String unnormalizedPath) throws Exception { File testDir = new File(path); FileUtils.touch(new File(testDir, "ab/cd/ef/abcdef")); FileUtils.touch(new File(testDir, "bc/de/fg/bcdefg")); FileUtils.touch(new File(testDir, "cd/ef/gh/cdefgh")); FileUtils.touch(new File(testDir, "c")); FileDataStore fds = new OakFileDataStore(); fds.setPath(unnormalizedPath); fds.init(null); Iterator<DataIdentifier> dis = fds.getAllIdentifiers(); Set<String> fileNames = Sets.newHashSet(Iterators.transform(dis, new Function<DataIdentifier, String>() { @Override public String apply(@Nullable DataIdentifier input) { return input.toString(); } })); Set<String> expectedNames = Sets.newHashSet("abcdef","bcdefg","cdefgh"); assertEquals(expectedNames, fileNames); FileUtils.cleanDirectory(testDir); }
public void testReference() throws Exception { byte[] data = new byte[12345]; new Random(12345).nextBytes(data); String reference; FileDataStore store = new FileDataStore(); store.init(testDir + "/reference"); try { DataRecord record = store.addRecord(new ByteArrayInputStream(data)); reference = record.getReference(); assertReference(data, reference, store); } finally { store.close(); } store = new FileDataStore(); store.init(testDir + "/reference"); try { assertReference(data, reference, store); } finally { store.close(); } }
public void testDeleteRecordWithParentCollision() throws Exception { FileDataStore fds = new FileDataStore(); fds.init(testDir + "/fileDeleteCollision"); ByteArrayInputStream c1 = new ByteArrayInputStream(ByteBuffer .allocate(8).putLong(0x181c7).array()); ByteArrayInputStream c2 = new ByteArrayInputStream(ByteBuffer .allocate(8).putLong(0x11fd78).array()); DataRecord d1 = fds.addRecord(c1); DataRecord d2 = fds.addRecord(c2); fds.deleteRecord(d1.getIdentifier()); DataRecord testRecord = fds.getRecordIfStored(d2.getIdentifier()); assertNotNull(testRecord); assertEquals(d2.getIdentifier(), testRecord.getIdentifier()); // Check the presence of the parent directory (relies on internal details of the FileDataStore) File parentDirD1 = new File( fds.getPath() + System.getProperty("file.separator") + d1.getIdentifier().toString().substring(0, 2)); assertTrue(parentDirD1.exists()); }
public void testFile() throws Exception { FileDataStore store = new FileDataStore(); store.setPath(TEST_DIR + "/fs"); store.init(TEST_DIR + "/fs"); store.setMinRecordLength(0); doTest(store); }
@Test public void readOnly() throws Exception{ FileDataStore fds = new FileDataStore(); fds.setPath(temporaryFolder.getRoot().getAbsolutePath()); fds.init(null); DataStoreBlobStore writableBS = new DataStoreBlobStore(fds); BlobStore readOnly = ReadOnlyBlobStoreWrapper.wrap(writableBS); try { readOnly.writeBlob(new ByteArrayInputStream("foo".getBytes())); fail(); } catch (Exception ignore) { } String blobId = writableBS.writeBlob(new ByteArrayInputStream("foo".getBytes())); try(InputStream is = readOnly.getInputStream(blobId)) { assertNotNull(is); } }
public void testDeleteRecordWithoutParentCollision() throws Exception { FileDataStore fds = new FileDataStore(); fds.init(testDir + "/fileDelete"); String c1 = "idhfigjhehgkdfgk"; String c2 = "02c60cb75083ceef"; DataRecord d1 = fds.addRecord(IOUtils.toInputStream(c1)); DataRecord d2 = fds.addRecord(IOUtils.toInputStream(c2)); fds.deleteRecord(d1.getIdentifier()); DataRecord testRecord = fds.getRecordIfStored(d2.getIdentifier()); assertNotNull(testRecord); assertEquals(d2.getIdentifier(), testRecord.getIdentifier()); // Check the absence of the parent directory (relies on internal details of the FileDataStore) File parentDirD1 = new File( fds.getPath() + System.getProperty("file.separator") + d1.getIdentifier().toString().substring(0, 2)); assertFalse(parentDirD1.exists()); }
@Test public void delegateRecordTest() throws Exception{ FileDataStore ds = new FileDataStore(); byte[] data = bytes(ds.getMinRecordLength() + 10); ds.init(tempFolder.getRoot().getAbsolutePath()); DataRecord dr = ds.addRecord(new ByteArrayInputStream(data)); File mappingFile = new File(tempFolder.getRoot(), "mapping.txt"); String text = String.format("%s|%s", data.length, dr.getIdentifier().toString()); Files.write(text, mappingFile, Charset.defaultCharset()); LengthCachingDataStore fds = new LengthCachingDataStore(); fds.setDelegateClass(FileDataStore.class.getName()); fds.setMappingFilePath(mappingFile.getAbsolutePath()); fds.init(tempFolder.getRoot().getAbsolutePath()); DataRecord dr2 = fds.getRecordIfStored(dr.getIdentifier()); assertEquals(dr, dr2); assertEquals(dr.getLength(), dr2.getLength()); assertTrue(supplier(dr).contentEquals(supplier(dr2))); }
private BlobStore createBlobStore() { FileDataStore fds = new OakFileDataStore(); fdsDir = "target/fds-" + codec + copyOnRW + minRecordLength + mergePolicy; fds.setPath(fdsDir); if (minRecordLength > 0) { fds.setMinRecordLength(minRecordLength); } fds.init(null); dataStoreBlobStore = new DataStoreBlobStore(fds); StatisticsProvider sp = new DefaultStatisticsProvider(scheduledExecutorService); BlobStatsCollector collector = new BlobStoreStats(sp); dataStoreBlobStore.setBlobStatsCollector(collector); return dataStoreBlobStore; }
@Override public Oak getOak(int clusterId) throws Exception { FileStoreBuilder fileStoreBuilder = fileStoreBuilder(parentPath) .withMaxFileSize(maxFileSize) .withSegmentCacheSize(segmentCacheSize) .withMemoryMapping(memoryMapping); if (azureConnectionString != null) { CloudStorageAccount cloud = CloudStorageAccount.parse(azureConnectionString); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(azureContainerName); container.createIfNotExists(); CloudBlobDirectory directory = container.getDirectoryReference(azureRootPath); fileStoreBuilder.withCustomPersistence(new AzurePersistence(directory)); } if (useBlobStore) { FileDataStore fds = new FileDataStore(); fds.setMinRecordLength(4092); fds.init(parentPath.getAbsolutePath()); BlobStore blobStore = new DataStoreBlobStore(fds); fileStoreBuilder.withBlobStore(blobStore); } FileStore fs = fileStoreBuilder.build(); return newOak(SegmentNodeStoreBuilders.builder(fs).build()); }
@Override public Oak getOak(int clusterId) throws Exception { FileStoreBuilder fileStoreBuilder = fileStoreBuilder(parentPath) .withMaxFileSize(maxFileSize) .withSegmentCacheSize(segmentCacheSize) .withMemoryMapping(memoryMapping); if (azureConnectionString != null) { CloudStorageAccount cloud = CloudStorageAccount.parse(azureConnectionString); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(azureContainerName); container.createIfNotExists(); CloudBlobDirectory directory = container.getDirectoryReference(azureRootPath); fileStoreBuilder.withCustomPersistence(new AzurePersistence(directory)); } if (useBlobStore) { FileDataStore fds = new FileDataStore(); fds.setMinRecordLength(4092); fds.init(parentPath.getAbsolutePath()); BlobStore blobStore = new DataStoreBlobStore(fds); fileStoreBuilder.withBlobStore(blobStore); } FileStore fs = fileStoreBuilder.build(); return newOak(SegmentNodeStoreBuilders.builder(fs).build()); }
fds.init(new File(tempFolder.getRoot(), "fdsRoot").getAbsolutePath()); DataStoreBlobStore dsbs = new DataStoreBlobStore(fds); BlobFactory factory = in -> new BlobStoreBlob(dsbs, dsbs.writeBlob(in));