public List<String> emptyCloudBlobDirectory(final String containerName, final String virtualDirPath) throws StorageException, URISyntaxException { List<String> deletedFiles = new ArrayList<>(); CloudBlobContainer container = getCloudBlobContainer(containerName); for (ListBlobItem blobItem : container.listBlobs(virtualDirPath, true, null, null, null)) { CloudBlob cloudBlob = (CloudBlob) blobItem; log.info("Removing file[%s] from Azure.", cloudBlob.getName()); if (cloudBlob.deleteIfExists()) { deletedFiles.add(cloudBlob.getName()); } } if (deletedFiles.isEmpty()) { log.warn("No files were deleted on the following Azure path: [%s]", virtualDirPath); } return deletedFiles; }
.blobName(cloudBlob.getName()) .containerName(containerName) .contentType(properties.getContentType())
public static String getName(CloudBlob blob) { return Paths.get(blob.getName()).getFileName().toString(); }
public static String getName(CloudBlob blob) { return Paths.get(blob.getName()).getFileName().toString(); }
@Override public Iterator<String> getNextObjectList(String prefix) throws IOException { List<String> al = new ArrayList<String>(); for (int i = 0; i < 1000; i++) { if (iter.hasNext()) { CloudBlob bi = (CloudBlob) iter.next(); // SDFSLogger.getLog().info("key name= " + bi.getName()); al.add(bi.getName()); } else return al.iterator(); } return al.iterator(); }
@Override public Map<String, Object> getReturnValues() { Map<String, Object> resultMap = super.getReturnValues(); resultMap.put(TAzureStorageContainerListDefinition.RETURN_TOTAL_RECORD_COUNT, dataCount); resultMap.put(AzureStorageContainerDefinition.RETURN_CONTAINER, properties.container.getValue()); if (currentBlob != null) { resultMap.put(AzureStorageBlobDefinition.RETURN_CURRENT_BLOB, currentBlob.getName()); } return resultMap; } }
@Override public Map<String, Object> getReturnValues() { Map<String, Object> resultMap = super.getReturnValues(); resultMap.put(TAzureStorageContainerListDefinition.RETURN_TOTAL_RECORD_COUNT, dataCount); resultMap.put(AzureStorageContainerDefinition.RETURN_CONTAINER, properties.container.getValue()); if (currentBlob != null) { resultMap.put(AzureStorageBlobDefinition.RETURN_CURRENT_BLOB, currentBlob.getName()); } return resultMap; } }
public String getNextName(String pp, long id) throws IOException { String pfx = pp + "/"; if (di == null) di = container.listBlobs(pp + "/").iterator(); while (di.hasNext()) { CloudBlob bi = (CloudBlob) di.next(); try { bi.downloadAttributes(); HashMap<String, String> md = bi.getMetadata(); boolean encrypt = Boolean.parseBoolean(md.get("encrypt")); String fname = EncyptUtils.decString(bi.getName().substring(pfx.length()), encrypt); return fname; /* * this.downloadFile(fname, new File(to.getPath() + File.separator + fname), * pp); */ } catch (Exception e) { throw new IOException(e); } } di = null; return null; }
@Override public ChunkData getNextChunck() throws IOException { try { if (!iter.hasNext()) { pool.returnObject(iContainer); return null; } else { CloudBlob bi = (CloudBlob) iter.next(); HashMap<String, String> md = bi.getMetadata(); boolean encrypt = Boolean.parseBoolean(md.get("encrypt")); byte[] nm = this.getHashBytes(bi.getName(), encrypt); int sz = Integer.parseInt(md.get("size")); int cl = Integer.parseInt(md.get("compressedsize")); ChunkData chk = new ChunkData(nm, 0); chk.cLen = sz; this.currentLength.addAndGet(chk.cLen); this.compressedLength.addAndGet(cl); return chk; } } catch (Exception e) { throw new IOException(e); } }
@Override public boolean advance() throws IOException { advanceable = blobsIterator.hasNext(); if (advanceable) { dataCount++; currentBlob = blobsIterator.next(); IndexedRecord dataRecord = new GenericData.Record(properties.schema.schema.getValue()); dataRecord.put(0, currentBlob.getName()); currentRecord.put(0, dataRecord); currentRecord.put(1, dataRecord); } return advanceable; }
public List<String> emptyCloudBlobDirectory(final String containerName, final String virtualDirPath) throws StorageException, URISyntaxException { List<String> deletedFiles = new ArrayList<>(); CloudBlobContainer container = getCloudBlobContainer(containerName); for (ListBlobItem blobItem : container.listBlobs(virtualDirPath, true, null, null, null)) { CloudBlob cloudBlob = (CloudBlob) blobItem; log.info("Removing file[%s] from Azure.", cloudBlob.getName()); if (cloudBlob.deleteIfExists()) { deletedFiles.add(cloudBlob.getName()); } } if (deletedFiles.isEmpty()) { log.warn("No files were deleted on the following Azure path: [%s]", virtualDirPath); } return deletedFiles; }
@Override public boolean advance() throws IOException { advanceable = blobsIterator.hasNext(); if (advanceable) { dataCount++; currentBlob = blobsIterator.next(); IndexedRecord dataRecord = new GenericData.Record(properties.schema.schema.getValue()); dataRecord.put(0, currentBlob.getName()); currentRecord.put(0, dataRecord); currentRecord.put(1, dataRecord); } return advanceable; }
public static AzureBlobInfo fromCloudBlob(CloudBlob cloudBlob) { return new AzureBlobInfo(cloudBlob.getName(), cloudBlob.getProperties().getLastModified().getTime(), cloudBlob.getProperties().getLength()); } }
@Override protected void removeAll(String clustername) { if (clustername == null) { return; } clustername = sanitize(clustername); Iterable<ListBlobItem> listBlobItems = containerReference.listBlobs(clustername); for (ListBlobItem blobItem : listBlobItems) { try { // If the item is a blob and not a virtual directory. if (blobItem instanceof CloudBlob) { CloudBlob blob = (CloudBlob) blobItem; boolean deleted = blob.deleteIfExists(); if (deleted) { log.trace("Deleted file '%s'.", blob.getName()); } else { log.debug("Tried to delete file '%s' but it was already deleted.", blob.getName()); } } } catch (Exception e) { log.error("Error deleting ping data for cluster '" + clustername + "'.", e); } } }
@Override protected void removeAll(String clustername) { if (clustername == null) { return; } clustername = sanitize(clustername); Iterable<ListBlobItem> listBlobItems = containerReference.listBlobs(clustername); for (ListBlobItem blobItem : listBlobItems) { try { // If the item is a blob and not a virtual directory. if (blobItem instanceof CloudBlob) { CloudBlob blob = (CloudBlob) blobItem; boolean deleted = blob.deleteIfExists(); if (deleted) { log.trace("Deleted file '%s'.", blob.getName()); } else { log.debug("Tried to delete file '%s' but it was already deleted.", blob.getName()); } } } catch (Exception e) { log.error("Error deleting ping data for cluster '" + clustername + "'.", e); } } }
@Override public CloudBlob preProcessResponse(CloudBlob blob, CloudBlobClient client, OperationContext context) throws Exception { if (this.getResult().getStatusCode() != HttpURLConnection.HTTP_CREATED) { this.setNonExceptionedRetryableFailure(true); return null; } CloudBlob snapshot = null; final String snapshotTime = BlobResponse.getSnapshotTime(this.getConnection()); if (blob instanceof CloudBlockBlob) { snapshot = new CloudBlockBlob(blob.getName(), snapshotTime, CloudBlob.this.getContainer()); } else if (blob instanceof CloudPageBlob) { snapshot = new CloudPageBlob(blob.getName(), snapshotTime, CloudBlob.this.getContainer()); } else if (blob instanceof CloudAppendBlob) { snapshot = new CloudAppendBlob(blob.getName(), snapshotTime, CloudBlob.this.getContainer()); } snapshot.setProperties(blob.properties); // use the specified metadata if not null : otherwise blob's metadata snapshot.setMetadata(metadata != null ? metadata : blob.metadata); snapshot.updateEtagAndLastModifiedFromResponse(this.getConnection()); return snapshot; } };
@Override public CloudBlob preProcessResponse(CloudBlob blob, CloudBlobClient client, OperationContext context) throws Exception { if (this.getResult().getStatusCode() != HttpURLConnection.HTTP_CREATED) { this.setNonExceptionedRetryableFailure(true); return null; } CloudBlob snapshot = null; final String snapshotTime = BlobResponse.getSnapshotTime(this.getConnection()); if (blob instanceof CloudBlockBlob) { snapshot = new CloudBlockBlob(blob.getName(), snapshotTime, CloudBlob.this.getContainer()); } else if (blob instanceof CloudPageBlob) { snapshot = new CloudPageBlob(blob.getName(), snapshotTime, CloudBlob.this.getContainer()); } else if (blob instanceof CloudAppendBlob) { snapshot = new CloudAppendBlob(blob.getName(), snapshotTime, CloudBlob.this.getContainer()); } snapshot.setProperties(blob.properties); // use the specified metadata if not null : otherwise blob's metadata snapshot.setMetadata(metadata != null ? metadata : blob.metadata); snapshot.updateEtagAndLastModifiedFromResponse(this.getConnection()); return snapshot; } };
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlobContainerReferenceFromServerSAS() throws StorageException, URISyntaxException, IOException, InvalidKeyException { this.container.create(); CloudBlob blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, null, 1024, null); SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); Calendar now = Calendar.getInstance(); now.add(Calendar.MINUTE, 10); policy.setSharedAccessExpiryTime(now.getTime()); policy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ)); String token = this.container.generateSharedAccessSignature(policy, null); CloudBlobContainer containerSAS = new CloudBlobContainer(this.container.getStorageUri(), new StorageCredentialsSharedAccessSignature(token)); CloudBlob blobRef = containerSAS.getBlobReferenceFromServer(blob.getName()); assertEquals(blob.getClass(), blobRef.getClass()); assertEquals(blob.getUri(), blobRef.getUri()); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlobContainerReferenceFromServer() throws StorageException, URISyntaxException, IOException { this.container.create(); CloudBlob blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, null, 1024, null); blob.getProperties().setContentType("application/octet-stream"); blob.getProperties().setLength(1024); CloudBlob blobRef = this.container.getBlobReferenceFromServer(blob.getName()); BlobTestHelper.assertAreEqual(blob, blobRef); blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.PAGE_BLOB, null, 1024, null); blob.getProperties().setContentType("application/octet-stream"); blob.getProperties().setLength(1024); blobRef = this.container.getBlobReferenceFromServer(blob.getName()); BlobTestHelper.assertAreEqual(blob, blobRef); blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.APPEND_BLOB, null, 1024, null); blob.getProperties().setContentType("application/octet-stream"); blob.getProperties().setLength(1024); blobRef = this.container.getBlobReferenceFromServer(blob.getName()); BlobTestHelper.assertAreEqual(blob, blobRef); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlobContainerReferenceFromServerSnapshot() throws StorageException, URISyntaxException, IOException { this.container.create(); CloudBlob blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, null, 1024, null); CloudBlob snapshot = blob.createSnapshot(); snapshot.getProperties().setContentType("application/octet-stream"); snapshot.getProperties().setLength(1024); CloudBlob blobRef = this.container.getBlobReferenceFromServer(snapshot.getName(), snapshot.getSnapshotID(), null, null, null); BlobTestHelper.assertAreEqual(snapshot, blobRef); }