private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } return new AzurePersistence(cloudBlobDirectory); }
private Stream<CloudBlob> getBlobs(String archiveName) throws IOException { return AzureUtilities.getBlobs(getDirectory(archiveName)); }
@Override public String getName() { return AzureUtilities.getName(archiveDirectory); }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { UUID uuid = new UUID(msb, lsb); Optional<SegmentWriteAction> segment = queue.map(q -> q.read(uuid)); if (segment.isPresent()) { return segment.get().toBuffer(); } AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); return buffer; }
public static String getSegmentFileName(AzureSegmentArchiveEntry indexEntry) { return getSegmentFileName(indexEntry.getPosition(), indexEntry.getMsb(), indexEntry.getLsb()); }
private Buffer readBlob(String name) throws IOException { try { CloudBlockBlob blob = getBlob(name); if (!blob.exists()) { return null; } long length = blob.getProperties().getLength(); Buffer buffer = Buffer.allocate((int) length); AzureUtilities.readBufferFully(blob, buffer); return buffer; } catch (StorageException e) { throw new IOException(e); } }
@Override public void clean() throws IOException { try { AzureUtilities.deleteAllEntries(container.getDirectoryReference(dir)); } catch (URISyntaxException e) { throw new IOException(e); } }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { UUID uuid = new UUID(msb, lsb); Optional<SegmentWriteAction> segment = queue.map(q -> q.read(uuid)); if (segment.isPresent()) { return segment.get().toBuffer(); } AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); return buffer; }
public static String getSegmentFileName(AzureSegmentArchiveEntry indexEntry) { return getSegmentFileName(indexEntry.getPosition(), indexEntry.getMsb(), indexEntry.getLsb()); }
private Buffer readBlob(String name) throws IOException { try { CloudBlockBlob blob = getBlob(name); if (!blob.exists()) { return null; } long length = blob.getProperties().getLength(); Buffer buffer = Buffer.allocate((int) length); AzureUtilities.readBufferFully(blob, buffer); return buffer; } catch (StorageException e) { throw new IOException(e); } }
private AzurePersistence createAzurePersistence() throws StorageException, URISyntaxException, InvalidKeyException { CloudBlobDirectory cloudBlobDirectory = null; if (accountName != null && uri != null) { String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = new StorageCredentialsAccountAndKey(accountName, key); cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } else if (connectionString != null && containerName != null) { cloudBlobDirectory = AzureUtilities.cloudBlobDirectoryFrom(connectionString, containerName, dir); } if (cloudBlobDirectory == null) { throw new IllegalArgumentException("Could not connect to Azure storage. Too few connection parameters specified!"); } return new AzurePersistence(cloudBlobDirectory); }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } ioMonitor.beforeSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength()); Stopwatch stopwatch = Stopwatch.createStarted(); readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); long elapsed = stopwatch.elapsed(TimeUnit.NANOSECONDS); ioMonitor.afterSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength(), elapsed); return buffer; }
@Override public String getName() { return AzureUtilities.getName(archiveDirectory); }
private Stream<CloudBlob> getBlobs(String archiveName) throws IOException { return AzureUtilities.getBlobs(getDirectory(archiveName)); }
private void doWriteEntry(AzureSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException { long msb = indexEntry.getMsb(); long lsb = indexEntry.getLsb(); ioMonitor.beforeSegmentWrite(pathAsFile(), msb, lsb, size); Stopwatch stopwatch = Stopwatch.createStarted(); try { CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry)); blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); blob.uploadFromByteArray(data, offset, size); blob.uploadMetadata(); } catch (StorageException e) { throw new IOException(e); } ioMonitor.afterSegmentWrite(pathAsFile(), msb, lsb, size, stopwatch.elapsed(TimeUnit.NANOSECONDS)); }
public static CloudBlobDirectory createCloudBlobDirectory(String path) { Map<String, String> config = parseAzureConfigurationFromUri(path); String accountName = config.get(KEY_ACCOUNT_NAME); String key = System.getenv("AZURE_SECRET_KEY"); StorageCredentials credentials = null; try { credentials = new StorageCredentialsAccountAndKey(accountName, key); } catch (IllegalArgumentException e) { throw new IllegalArgumentException( "Could not connect to the Azure Storage. Please verify if AZURE_SECRET_KEY environment variable " + "is correctly set!"); } String uri = config.get(KEY_STORAGE_URI); String dir = config.get(KEY_DIR); try { return AzureUtilities.cloudBlobDirectoryFrom(credentials, uri, dir); } catch (URISyntaxException | StorageException e) { throw new IllegalArgumentException( "Could not connect to the Azure Storage. Please verify the path provided!"); } }
@Override public Buffer readSegment(long msb, long lsb) throws IOException { AzureSegmentArchiveEntry indexEntry = index.get(new UUID(msb, lsb)); if (indexEntry == null) { return null; } Buffer buffer; if (OFF_HEAP) { buffer = Buffer.allocateDirect(indexEntry.getLength()); } else { buffer = Buffer.allocate(indexEntry.getLength()); } ioMonitor.beforeSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength()); Stopwatch stopwatch = Stopwatch.createStarted(); readBufferFully(getBlob(getSegmentFileName(indexEntry)), buffer); long elapsed = stopwatch.elapsed(TimeUnit.NANOSECONDS); ioMonitor.afterSegmentRead(pathAsFile(), msb, lsb, indexEntry.getLength(), elapsed); return buffer; }
@Override public String getName() { return AzureUtilities.getName(archiveDirectory); }
public static void deleteAllEntries(CloudBlobDirectory directory) throws IOException { Stream<CloudBlob> blobs = getBlobs(directory); blobs.forEach(b -> { try { b.deleteIfExists(); } catch (StorageException e) { log.error("Can't delete blob {}", b.getUri().getPath(), e); } }); }
private void doWriteEntry(AzureSegmentArchiveEntry indexEntry, byte[] data, int offset, int size) throws IOException { long msb = indexEntry.getMsb(); long lsb = indexEntry.getLsb(); ioMonitor.beforeSegmentWrite(pathAsFile(), msb, lsb, size); Stopwatch stopwatch = Stopwatch.createStarted(); try { CloudBlockBlob blob = getBlob(getSegmentFileName(indexEntry)); blob.setMetadata(AzureBlobMetadata.toSegmentMetadata(indexEntry)); blob.uploadFromByteArray(data, offset, size); blob.uploadMetadata(); } catch (StorageException e) { throw new IOException(e); } ioMonitor.afterSegmentWrite(pathAsFile(), msb, lsb, size, stopwatch.elapsed(TimeUnit.NANOSECONDS)); }