private void checkValidKey(String key, String fieldUsed) { if (key == null || key.matches(REGEX_INVALID_KEY)) { throw new AzureException(String.format("Cannot insert to Azure Checkpoint Manager; %s %s contains invalid characters [*, /, \\\\, ?]", fieldUsed, key)); } }
public TableUtils(AzureClient client, String tableName, String initialState) { this.initialState = initialState; CloudTableClient tableClient = client.getTableClient(); try { table = tableClient.getTableReference(tableName); table.createIfNotExists(); } catch (URISyntaxException e) { LOG.error("\nConnection string specifies an invalid URI.", e); throw new AzureException(e); } catch (StorageException e) { LOG.error("Azure storage exception.", e); throw new AzureException(e); } }
@Override public void start() { try { // Create the table if it doesn't exist. cloudTable = azureClient.getTableClient().getTableReference(jobTableName); cloudTable.createIfNotExists(); } catch (URISyntaxException e) { LOG.error("Connection string {} specifies an invalid URI while creating checkpoint table.", storageConnectionString); throw new AzureException(e); } catch (StorageException e) { LOG.error("Azure Storage failed when creating checkpoint table", e); throw new AzureException(e); } }
AzureCheckpointManager(AzureConfig azureConfig, Option<String> jobName) { if (!jobName.isDefined()) { throw new AzureException("Jobs must have a name to use Azure Checkpoint Manager"); } // Remove invalid characters jobTableName = jobName.get().replaceAll(REGEX_TABLE_NAME, ""); storageConnectionString = azureConfig.getAzureConnectionString(); azureClient = new AzureClient(storageConnectionString); }
/** * Acquires a lease on a blob. The lease ID is NULL initially. * @param leaseTimeInSec The time in seconds you want to acquire the lease for. * @param leaseId Proposed ID you want to acquire the lease with, null if not proposed. * @return String that represents lease ID. Null if acquireLease is unsuccessful because the blob is leased already. * @throws AzureException If a Azure storage service error occurred. This includes the case where the blob you're trying to lease does not exist. */ public String acquireLease(int leaseTimeInSec, String leaseId) { try { String id = leaseBlob.acquireLease(leaseTimeInSec, leaseId); LOG.info("Acquired lease with lease id = " + id); return id; } catch (StorageException storageException) { int httpStatusCode = storageException.getHttpStatusCode(); if (httpStatusCode == HttpStatus.CONFLICT_409) { LOG.info("The blob you're trying to acquire is leased already.", storageException.getMessage()); } else if (httpStatusCode == HttpStatus.NOT_FOUND_404) { LOG.error("The blob you're trying to lease does not exist.", storageException); throw new AzureException(storageException); } else { LOG.error("Error acquiring lease!", storageException); throw new AzureException(storageException); } } return null; }
private JobModelBundle getJobModelBundle() { byte[] data = new byte[(int) JOB_MODEL_BLOCK_SIZE]; try { blob.downloadRangeToByteArray(0, JOB_MODEL_BLOCK_SIZE, data, 0); } catch (StorageException e) { LOG.error("Failed to read JobModel details from the blob.", e); throw new AzureException(e); } try { JobModelBundle jmBundle = SamzaObjectMapper.getObjectMapper().readValue(data, JobModelBundle.class); return jmBundle; } catch (IOException e) { LOG.error("Failed to parse byte data: " + data + " for JobModel details retrieved from the blob", e); throw new SamzaException(e); } }
/** * Reads the list of live processors published on the blob. * @return String list of live processors. * @throws AzureException If an Azure storage service error occurred. * @throws SamzaException If data retrieved from blob could not be parsed by SamzaObjectMapper. */ public List<String> getLiveProcessorList() { LOG.info("Read the the list of live processors from blob."); byte[] data = new byte[(int) PROCESSOR_LIST_BLOCK_SIZE]; try { blob.downloadRangeToByteArray(JOB_MODEL_BLOCK_SIZE + BARRIER_STATE_BLOCK_SIZE, PROCESSOR_LIST_BLOCK_SIZE, data, 0); } catch (StorageException e) { LOG.error("Failed to read the list of live processors from the blob.", new AzureException(e)); throw new AzureException(e); } List<String> list; try { list = SamzaObjectMapper.getObjectMapper().readValue(data, List.class); } catch (IOException e) { LOG.error("Failed to parse byte data: " + data + " for live processor list retrieved from the blob", new SamzaException(e)); throw new SamzaException(e); } return list; }
/** * Reads the current barrier state from the blob. * @return Barrier state published on the blob. * @throws AzureException If an Azure storage service error occurred. * @throws SamzaException If data retrieved from blob could not be parsed by SamzaObjectMapper. */ public String getBarrierState() { LOG.info("Reading the barrier state from blob."); byte[] data = new byte[(int) BARRIER_STATE_BLOCK_SIZE]; try { blob.downloadRangeToByteArray(JOB_MODEL_BLOCK_SIZE, BARRIER_STATE_BLOCK_SIZE, data, 0); } catch (StorageException e) { LOG.error("Failed to read barrier state from blob.", e); throw new AzureException(e); } String state; try { state = SamzaObjectMapper.getObjectMapper().readValue(data, String.class); } catch (IOException e) { LOG.error("Failed to parse byte data: " + data + " for barrier state retrieved from the blob.", e); throw new SamzaException(e); } return state; }
/** * Retrieve a particular row in the processor table, given the partition key and the row key. * @param jmVersion Job model version of the processor row to be retrieved. * @param pid Unique processor ID of the processor row to be retrieved. * @return An instance of required processor entity. Null if does not exist. * @throws AzureException If an Azure storage service error occurred. */ public ProcessorEntity getEntity(String jmVersion, String pid) { try { TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class); return table.execute(retrieveEntity).getResultAsType(); } catch (StorageException e) { LOG.error("Azure storage exception while retrieving processor entity with job model version: " + jmVersion + "and pid: " + pid, e); throw new AzureException(e); } }
private void deleteEntities(Iterator<TaskCheckpointEntity> entitiesToDelete) { TableBatchOperation batchOperation = new TableBatchOperation(); while (entitiesToDelete.hasNext()) { TaskCheckpointEntity entity = entitiesToDelete.next(); // Add to batch operation batchOperation.delete(entity); // Execute when batch reaches capacity or when this is the last item if (batchOperation.size() >= MAX_WRITE_BATCH_SIZE || !entitiesToDelete.hasNext()) { try { cloudTable.execute(batchOperation); } catch (StorageException e) { LOG.error("Executing batch failed for deleting checkpoints"); throw new AzureException(e); } batchOperation.clear(); } } } }
/** * Deletes a specified row in the processor table. * * Note: Table service uses optimistic locking by default. In order to disable it, set the ETag on the ProcessorEntity * to "*" before invoking this method. * * @param entity ProcessorEntity that has to be deleted * @throws AzureException If an Azure storage service error occurred. */ public void deleteProcessorEntity(ProcessorEntity entity) { try { TableOperation remove = TableOperation.delete(entity); table.execute(remove); } catch (StorageException e) { LOG.error("Azure storage exception while deleting processor entity with job model version: " + entity.getJobModelVersion() + "and pid: " + entity.getProcessorId(), e); throw new AzureException(e); } } /**
/** * Creates an object of BlobUtils. It creates the container and page blob if they don't exist already. * @param client Client handle for access to Azure Storage account. * @param containerName Name of container inside which we want the blob to reside. * @param blobName Name of the blob to be managed. * @param length Length of the page blob. * @throws AzureException If an Azure storage service error occurred, or when the container name or blob name is invalid. */ public BlobUtils(AzureClient client, String containerName, String blobName, long length) { this.blobClient = client.getBlobClient(); try { this.container = blobClient.getContainerReference(containerName); container.createIfNotExists(); this.blob = container.getPageBlobReference(blobName); if (!blob.exists()) { blob.create(length, AccessCondition.generateIfNotExistsCondition(), null, null); } } catch (URISyntaxException e) { LOG.error("Container name: " + containerName + " or blob name: " + blobName + " invalid.", e); throw new AzureException(e); } catch (StorageException e) { int httpStatusCode = e.getHttpStatusCode(); if (httpStatusCode == HttpStatus.CONFLICT_409) { LOG.info("The blob you're trying to create exists already.", e); } else { LOG.error("Azure Storage Exception!", e); throw new AzureException(e); } } }
/** * Add a row which denotes an active processor to the processor table. * @param jmVersion Job model version that the processor is operating on. * @param pid Unique processor ID. * @param isLeader Denotes whether the processor is a leader or not. * @throws AzureException If an Azure storage service error occurred. */ public void addProcessorEntity(String jmVersion, String pid, boolean isLeader) { ProcessorEntity entity = new ProcessorEntity(jmVersion, pid); entity.setIsLeader(isLeader); entity.updateLiveness(); TableOperation add = TableOperation.insert(entity); try { table.execute(add); } catch (StorageException e) { LOG.error("Azure storage exception while adding processor entity with job model version: " + jmVersion + "and pid: " + pid, e); throw new AzureException(e); } }
} catch (StorageException e) { LOG.error("Executing batch failed for task: {}", taskName); throw new AzureException(e);
LOG.error("Connection string {} specifies an invalid URI.", storageConnectionString); LOG.error("Please confirm the connection string is in the Azure connection string format."); throw new AzureException(e); } catch (InvalidKeyException e) { LOG.error("Connection string {} specifies an invalid key.", storageConnectionString); LOG.error("Please confirm the AccountName and AccountKey in the connection string are valid."); throw new AzureException(e);
/** * Updates the isLeader value when the processor starts or stops being a leader. * @param jmVersion Job model version of the processor row to be updated. * @param pid Unique processor ID of the processor row to be updated. * @param isLeader Denotes whether the processor is a leader or not. * @throws AzureException If an Azure storage service error occurred. */ public void updateIsLeader(String jmVersion, String pid, boolean isLeader) { try { TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class); ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType(); entity.setIsLeader(isLeader); TableOperation update = TableOperation.replace(entity); table.execute(update); } catch (StorageException e) { LOG.error("Azure storage exception while updating isLeader value for job model version: " + jmVersion + "and pid: " + pid, e); throw new AzureException(e); } }
/** * Deletes a specified row in the processor table. * * Note: Table service uses optimistic locking by default. Hence, if there is an update after retrieving the entity, * then the delete operation will fail. * * @param jmVersion Job model version of the processor row to be deleted. * @param pid Unique processor ID of the processor row to be deleted. * @param force True, to disable optimistic locking on the table. False, otherwise. Setting to false may result in * AzureException when there is concurrent access to the table. * * @throws AzureException If an Azure storage service error occurred. */ public void deleteProcessorEntity(String jmVersion, String pid, boolean force) { try { TableOperation retrieveEntity = TableOperation.retrieve(jmVersion, pid, ProcessorEntity.class); ProcessorEntity entity = table.execute(retrieveEntity).getResultAsType(); if (force) { entity.setEtag("*"); } TableOperation remove = TableOperation.delete(entity); table.execute(remove); } catch (StorageException e) { LOG.error("Azure storage exception while deleting processor entity with job model version: " + jmVersion + "and pid: " + pid, e); throw new AzureException(e); } }