public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, String pathOrUri) { SegmentNodeStorePersistence persistence = null; switch (storeType) { case AZURE: CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); persistence = new AzurePersistence(cloudBlobDirectory); break; default: persistence = new TarPersistence(new File(pathOrUri)); } return persistence; }
@Override public GCJournalFile getGCJournalFile() throws IOException { return new AzureGCJournalFile(getAppendBlob("gc.log")); }
@Override public ManifestFile getManifestFile() throws IOException { return new AzureManifestFile(getBlockBlob("manifest")); }
@Before @Override public void setUp() throws IOException { try { container = azurite.getContainer("oak-test"); archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter()); } catch (StorageException | InvalidKeyException | URISyntaxException e) { throw new IOException(e); } }
@Test public void testManifest() throws URISyntaxException, IOException { ManifestFile manifestFile = new AzurePersistence(container.getDirectoryReference("oak")).getManifestFile(); assertFalse(manifestFile.exists()); Properties props = new Properties(); props.setProperty("xyz", "abc"); props.setProperty("version", "123"); manifestFile.save(props); Properties loaded = manifestFile.load(); assertEquals(props, loaded); }
@Before @Override public void setUp() throws IOException { try { monitor = new TestFileStoreMonitor(); container = azurite.getContainer("oak-test"); archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, false, new IOMonitorAdapter(), monitor); } catch (StorageException | InvalidKeyException | URISyntaxException e) { throw new IOException(e); } } }
public static SegmentNodeStorePersistence newSegmentNodeStorePersistence(SegmentStoreType storeType, String pathOrUri) { SegmentNodeStorePersistence persistence = null; switch (storeType) { case AZURE: CloudBlobDirectory cloudBlobDirectory = createCloudBlobDirectory(pathOrUri.substring(3)); persistence = new AzurePersistence(cloudBlobDirectory); break; default: persistence = new TarPersistence(new File(pathOrUri)); } return persistence; }
@Test public void testRecovery() throws StorageException, URISyntaxException, IOException { SegmentArchiveManager manager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(false, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter()); SegmentArchiveWriter writer = manager.create("data00000a.tar"); List<UUID> uuids = new ArrayList<>(); for (int i = 0; i < 10; i++) { UUID u = UUID.randomUUID(); writer.writeSegment(u.getMostSignificantBits(), u.getLeastSignificantBits(), new byte[10], 0, 10, 0, 0, false); uuids.add(u); } writer.flush(); writer.close(); container.getBlockBlobReference("oak/data00000a.tar/0005." + uuids.get(5).toString()).delete(); LinkedHashMap<UUID, byte[]> recovered = new LinkedHashMap<>(); manager.recoverEntries("data00000a.tar", recovered); assertEquals(uuids.subList(0, 5), newArrayList(recovered.keySet())); }
@Override public ManifestFile getManifestFile() throws IOException { return new AzureManifestFile(getBlockBlob("manifest")); }
@Override public GCJournalFile getGCJournalFile() throws IOException { return new AzureGCJournalFile(getAppendBlob("gc.log")); }
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } return new AzurePersistence(cloudBlobDirectory); }
@Override public RepositoryLock lockRepository() throws IOException { return new AzureRepositoryLock(getBlockBlob("repo.lock"), () -> { log.warn("Lost connection to the Azure. The client will be closed."); // TODO close the connection }).lock(); }
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } return new AzurePersistence(cloudBlobDirectory); }
@Override public RepositoryLock lockRepository() throws IOException { return new AzureRepositoryLock(getBlockBlob("repo.lock"), () -> { log.warn("Lost connection to the Azure. The client will be closed."); // TODO close the connection }).lock(); }
@Override protected SegmentNodeStorePersistence getPersistence() throws IOException { try { return new AzurePersistence(container.getDirectoryReference("oak")); } catch (URISyntaxException e) { throw new IOException(e); } } }
@Override protected SegmentNodeStorePersistence getPersistence() throws Exception { return new AzurePersistence(container.getDirectoryReference("oak")); }
private static SegmentNodeStorePersistence createAzurePersistence(Configuration configuration) throws IOException { try { StringBuilder connectionString = new StringBuilder(); if (configuration.connectionURL() == null || configuration.connectionURL().trim().isEmpty()) { connectionString.append("DefaultEndpointsProtocol=https;"); connectionString.append("AccountName=").append(configuration.accountName()).append(';'); connectionString.append("AccountKey=").append(configuration.accessKey()).append(';'); } else { connectionString.append(configuration.connectionURL()); } log.info("Connection string: '{}'", connectionString.toString()); CloudStorageAccount cloud = CloudStorageAccount.parse(connectionString.toString()); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(configuration.containerName()); container.createIfNotExists(); String path = configuration.rootPath(); if (path != null && path.length() > 0 && path.charAt(0) == '/') { path = path.substring(1); } AzurePersistence persistence = new AzurePersistence(container.getDirectoryReference(path)); return persistence; } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new IOException(e); } }
private static SegmentNodeStorePersistence createAzurePersistence(Configuration configuration) throws IOException { try { StringBuilder connectionString = new StringBuilder(); if (configuration.connectionURL() == null || configuration.connectionURL().trim().isEmpty()) { connectionString.append("DefaultEndpointsProtocol=https;"); connectionString.append("AccountName=").append(configuration.accountName()).append(';'); connectionString.append("AccountKey=").append(configuration.accessKey()).append(';'); } else { connectionString.append(configuration.connectionURL()); } log.info("Connection string: '{}'", connectionString.toString()); CloudStorageAccount cloud = CloudStorageAccount.parse(connectionString.toString()); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(configuration.containerName()); container.createIfNotExists(); String path = configuration.rootPath(); if (path != null && path.length() > 0 && path.charAt(0) == '/') { path = path.substring(1); } AzurePersistence persistence = new AzurePersistence(container.getDirectoryReference(path)); return persistence; } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new IOException(e); } }
public Builder withSource(CloudBlobDirectory dir) throws URISyntaxException, StorageException { this.source = new AzurePersistence(dir); this.sourceName = storeDescription(SegmentStoreType.AZURE, dir.getContainer().getName() + "/" + dir.getPrefix()); return this; }
public Builder withTarget(CloudBlobDirectory dir) throws URISyntaxException, StorageException { this.target = new AzurePersistence(dir); this.targetName = storeDescription(SegmentStoreType.AZURE, dir.getContainer().getName() + "/" + dir.getPrefix()); return this; }