public AzureUploadManager(SecorConfig config) throws Exception { super(config); final String storageConnectionString = "DefaultEndpointsProtocol=" + mConfig.getAzureEndpointsProtocol() + ";" + "AccountName=" + mConfig.getAzureAccountName() + ";" + "AccountKey=" + mConfig.getAzureAccountKey() + ";"; CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); blobClient = storageAccount.createCloudBlobClient(); }
public static void setProxy(final OperationContext operationContext, final ProcessContext processContext) { final ProxyConfiguration proxyConfig = ProxyConfiguration.getConfiguration(processContext); operationContext.setProxy(proxyConfig.createProxy()); } }
public void init() { try { String endpoint = String.format(ENDPOINT_TEMPLATE, configuration.getAccountName()); CloudStorageAccount cosmosAccount = CloudStorageAccount.parse( String.format( COSMOS_CONNECTION_TEMPLATE, configuration.getAccountName(), configuration.getAccountKey(), endpoint)); tableClient = cosmosAccount.createCloudTableClient(); // Create the tables if the do not exist tableClient.getTableReference(JOB_TABLE).createIfNotExists(); tableClient.getTableReference(JOB_DATA_TABLE).createIfNotExists(); CloudStorageAccount blobAccount = CloudStorageAccount.parse( String.format( BLOB_CONNECTION_TEMPLATE, configuration.getAccountName(), configuration.getBlobKey())); blobClient = blobAccount.createCloudBlobClient(); blobClient.getContainerReference(BLOB_CONTAINER).createIfNotExists(); } catch (StorageException | URISyntaxException | InvalidKeyException e) { throw new MicrosoftStorageException(e); } }
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null); CloudBlobContainer container = blobClient.getContainerReference(containerName); final OperationContext operationContext = new OperationContext(); AzureStorageUtils.setProxy(operationContext, context); for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, operationContext)) { if (blob instanceof CloudBlob) { CloudBlob cloudBlob = (CloudBlob) blob; BlobProperties properties = cloudBlob.getProperties(); StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri(); .primaryUri(uri.getPrimaryUri().toString()) .blobName(cloudBlob.getName()) .containerName(containerName) .contentType(properties.getContentType()) .contentLanguage(properties.getContentLanguage()) .etag(properties.getEtag()) .lastModifiedTime(properties.getLastModified().getTime()) .length(properties.getLength()); if (uri.getSecondaryUri() != null) { builder.secondaryUri(uri.getSecondaryUri().toString());
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); final OperationContext operationContext = new OperationContext(); AzureStorageUtils.setProxy(operationContext, context); final CloudBlob blob = container.getBlockBlobReference(blobPath); blob.download(os, null, null, operationContext); } catch (StorageException e) { storedException.set(e); long length = blob.getProperties().getLength(); attributes.put("azure.length", String.valueOf(length)); session.getProvenanceReporter().fetch(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis); } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) { if (e instanceof ProcessException && storedException.get() == null) {
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); final OperationContext operationContext = new OperationContext(); AzureStorageUtils.setProxy(operationContext, context); blob.upload(in, length, null, null, operationContext); BlobProperties properties = blob.getProperties(); attributes.put("azure.container", containerName); attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString()); attributes.put("azure.etag", properties.getEtag()); attributes.put("azure.length", String.valueOf(length)); attributes.put("azure.timestamp", String.valueOf(properties.getLastModified())); } catch (StorageException | URISyntaxException e) { storedException.set(e); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); final OperationContext operationContext = new OperationContext(); AzureStorageUtils.setProxy(operationContext, context); blob.deleteIfExists(DeleteSnapshotsOption.valueOf(deleteSnapshotOptions), null, null, operationContext); session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis); } catch ( StorageException | URISyntaxException e) { getLogger().error("Failed to delete the specified blob {} from Azure Storage. Routing to failure", new Object[]{blobPath}, e);
StorageCredentials creds = new StorageCredentialsSharedAccessSignature(sasToken); cloudBlobClient = new CloudBlobClient(new URI(storageConnectionString), creds); } else { String blobConnString = String.format(AzureStorageUtils.FORMAT_BLOB_CONNECTION_STRING, accountName, accountKey); CloudStorageAccount storageAccount = CloudStorageAccount.parse(blobConnString); cloudBlobClient = storageAccount.createCloudBlobClient();
String storageConnectionString = getStorageConnectionString(protocol, account, key, tableEndPoint); try { storageAccount = CloudStorageAccount.parse(storageConnectionString); } catch (Exception e) { throw new DBException("Could not connect to the account.\n", e); tableClient = storageAccount.createCloudTableClient(); try { cloudTable = tableClient.getTableReference(table); cloudTable.createIfNotExists(); } catch (Exception e) { throw new DBException("Could not connect to the table.\n", e);
cloudQueue = cloudQueueClient.getQueueReference(queue); final OperationContext operationContext = new OperationContext(); AzureStorageUtils.setProxy(operationContext, context); retrievedMessagesIterable = cloudQueue.retrieveMessages(batchSize, visibilityTimeoutInSecs, null, operationContext); } catch (URISyntaxException | StorageException e) { getLogger().error("Failed to retrieve messages from the provided Azure Storage Queue due to {}", new Object[] {e}); attributes.put("azure.queue.uri", cloudQueue.getUri().toString()); attributes.put("azure.queue.insertionTime", message.getInsertionTime().toString()); attributes.put("azure.queue.expirationTime", message.getExpirationTime().toString()); attributes.put("azure.queue.messageId", message.getMessageId()); attributes.put("azure.queue.popReceipt", message.getPopReceipt()); out.write(message.getMessageContentAsByte()); } catch (StorageException e) { getLogger().error("Failed to write the retrieved queue message to FlowFile content due to {}", new Object[] {e}); session.getProvenanceReporter().receive(flowFile, cloudQueue.getStorageUri().toString()); cloudQueue.deleteMessage(message); } catch (StorageException e) { getLogger().error("Failed to delete the retrieved message with the id {} from the queue due to {}", new Object[] {message.getMessageId(), e});
if (StringUtils.isNoneBlank(sasToken)) { connectionString = String.format(FORMAT_QUEUE_BASE_URI, storageAccountName); StorageCredentials storageCredentials = new StorageCredentialsSharedAccessSignature(sasToken); cloudQueueClient = new CloudQueueClient(new URI(connectionString), storageCredentials); } else { connectionString = String.format(FORMAT_QUEUE_CONNECTION_STRING, storageAccountName, storageAccountKey); CloudStorageAccount storageAccount = CloudStorageAccount.parse(connectionString); cloudQueueClient = storageAccount.createCloudQueueClient();
final String flowFileContent = baos.toString(); CloudQueueMessage message = new CloudQueueMessage(flowFileContent); CloudQueueClient cloudQueueClient; CloudQueue cloudQueue; cloudQueue = cloudQueueClient.getQueueReference(queue); final OperationContext operationContext = new OperationContext(); AzureStorageUtils.setProxy(operationContext, context); cloudQueue.addMessage(message, ttl, delay, null, operationContext); } catch (URISyntaxException | StorageException e) { getLogger().error("Failed to write the message to Azure Queue Storage due to {}", new Object[]{e}); session.getProvenanceReporter().send(flowFile, cloudQueue.getUri().toString(), transmissionMillis);
/** * Initializes a new instances of the ServiceProperties class. */ public ServiceProperties() { this.setLogging(new LoggingProperties()); this.setHourMetrics(new MetricsProperties()); this.setMinuteMetrics(new MetricsProperties()); this.setCors(new CorsProperties()); // DeleteRetentionPolicy and StaticWebsiteProperties are not included by default because they are not accepted by services other than blob. }
/** * Fires events representing that a request will be sent. */ private static void fireSendingRequestEvent(OperationContext opContext, HttpURLConnection request, RequestResult result) { if (opContext.getSendingRequestEventHandler().hasListeners() || OperationContext.getGlobalSendingRequestEventHandler().hasListeners()) { SendingRequestEvent event = new SendingRequestEvent(opContext, request, result); opContext.getSendingRequestEventHandler().fireEvent(event); OperationContext.getGlobalSendingRequestEventHandler().fireEvent(event); } }
@Override public void kill(DataSegment segment) throws SegmentLoadingException { log.info("Killing segment [%s]", segment); Map<String, Object> loadSpec = segment.getLoadSpec(); final String containerName = MapUtils.getString(loadSpec, "containerName"); final String blobPath = MapUtils.getString(loadSpec, "blobPath"); final String dirPath = Paths.get(blobPath).getParent().toString(); try { azureStorage.emptyCloudBlobDirectory(containerName, dirPath); } catch (StorageException e) { Object extendedInfo = e.getExtendedErrorInformation() == null ? null : e.getExtendedErrorInformation().getErrorMessage(); throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getId(), extendedInfo); } catch (URISyntaxException e) { throw new SegmentLoadingException(e, "Couldn't kill segment[%s]: [%s]", segment.getId(), e.getReason()); } }
/** * Fires events representing that a response has been received. */ private static void fireResponseReceivedEvent(OperationContext opContext, HttpURLConnection request, RequestResult result) { if (opContext.getResponseReceivedEventHandler().hasListeners() || OperationContext.getGlobalResponseReceivedEventHandler().hasListeners()) { ResponseReceivedEvent event = new ResponseReceivedEvent(opContext, request, result); opContext.getResponseReceivedEventHandler().fireEvent(event); OperationContext.getGlobalResponseReceivedEventHandler().fireEvent(event); } }
/** * Fires events representing that an error occurred when receiving the response. */ private static void fireErrorReceivingResponseEvent(OperationContext opContext, HttpURLConnection request, RequestResult result) { if (opContext.getErrorReceivingResponseEventHandler().hasListeners() || OperationContext.getGlobalErrorReceivingResponseEventHandler().hasListeners()) { ErrorReceivingResponseEvent event = new ErrorReceivingResponseEvent(opContext, request, result); opContext.getErrorReceivingResponseEventHandler().fireEvent(event); OperationContext.getGlobalErrorReceivingResponseEventHandler().fireEvent(event); } }
/** * Fires events representing that a response received from the service is fully processed. */ private static void fireRequestCompletedEvent(OperationContext opContext, HttpURLConnection request, RequestResult result) { if (opContext.getRequestCompletedEventHandler().hasListeners() || OperationContext.getGlobalRequestCompletedEventHandler().hasListeners()) { RequestCompletedEvent event = new RequestCompletedEvent(opContext, request, result); opContext.getRequestCompletedEventHandler().fireEvent(event); OperationContext.getGlobalRequestCompletedEventHandler().fireEvent(event); } }
@Provides @LazySingleton public CloudBlobClient getCloudBlobClient(final AzureAccountConfig config) throws URISyntaxException, InvalidKeyException { CloudStorageAccount account = CloudStorageAccount.parse( StringUtils.format( STORAGE_CONNECTION_STRING, config.getProtocol(), config.getAccount(), config.getKey() ) ); return account.createCloudBlobClient(); }
/** * Initializes a new instances of the ServiceProperties class. */ public ServiceProperties() { this.setLogging(new LoggingProperties()); this.setHourMetrics(new MetricsProperties()); this.setMinuteMetrics(new MetricsProperties()); this.setCors(new CorsProperties()); }