System.out.println("retentionExpirationTime: " + new Date(blob.getRetentionExpirationTime())); if (blob.getMetadata() != null) { System.out.println("\n\n\nUser metadata:"); for (Map.Entry<String, String> userMetadata : blob.getMetadata().entrySet()) { System.out.println(userMetadata.getKey() + "=" + userMetadata.getValue());
@Test public void testUpdateBlobUnsetMetadata() { String blobName = "test-update-blob-unset-metadata"; ImmutableMap<String, String> metadata = ImmutableMap.of("k1", "a", "k2", "b"); Map<String, String> newMetadata = new HashMap<>(); newMetadata.put("k1", "a"); newMetadata.put("k2", null); ImmutableMap<String, String> expectedMetadata = ImmutableMap.of("k1", "a"); BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) .build(); Blob remoteBlob = storage.create(blob); assertNotNull(remoteBlob); Blob updatedBlob = remoteBlob.toBuilder().setMetadata(newMetadata).build().update(); assertNotNull(updatedBlob); assertEquals(blob.getName(), updatedBlob.getName()); assertEquals(blob.getBucket(), updatedBlob.getBucket()); assertEquals(expectedMetadata, updatedBlob.getMetadata()); }
@Test public void testGetBlobSelectedFields() { String blobName = "test-get-selected-fields-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .setMetadata(ImmutableMap.of("k", "v")) .build(); assertNotNull(storage.create(blob)); Blob remoteBlob = storage.get(blob.getBlobId(), Storage.BlobGetOption.fields(BlobField.METADATA)); assertEquals(blob.getBlobId(), remoteBlob.getBlobId()); assertEquals(ImmutableMap.of("k", "v"), remoteBlob.getMetadata()); assertNull(remoteBlob.getContentType()); }
@Test public void testUpdateBlobMergeMetadata() { String blobName = "test-update-blob-merge-metadata"; ImmutableMap<String, String> metadata = ImmutableMap.of("k1", "a"); ImmutableMap<String, String> newMetadata = ImmutableMap.of("k2", "b"); ImmutableMap<String, String> expectedMetadata = ImmutableMap.of("k1", "a", "k2", "b"); BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) .build(); Blob remoteBlob = storage.create(blob); assertNotNull(remoteBlob); Blob updatedBlob = remoteBlob.toBuilder().setMetadata(newMetadata).build().update(); assertNotNull(updatedBlob); assertEquals(blob.getName(), updatedBlob.getName()); assertEquals(blob.getBucket(), updatedBlob.getBucket()); assertEquals(expectedMetadata, updatedBlob.getMetadata()); }
@Test public void testUpdateBlobReplaceMetadata() { String blobName = "test-update-blob-replace-metadata"; ImmutableMap<String, String> metadata = ImmutableMap.of("k1", "a"); ImmutableMap<String, String> newMetadata = ImmutableMap.of("k2", "b"); BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) .build(); Blob remoteBlob = storage.create(blob); assertNotNull(remoteBlob); Blob updatedBlob = remoteBlob.toBuilder().setMetadata(null).build().update(); assertNotNull(updatedBlob); assertNull(updatedBlob.getMetadata()); updatedBlob = remoteBlob.toBuilder().setMetadata(newMetadata).build().update(); assertEquals(blob.getName(), updatedBlob.getName()); assertEquals(blob.getBucket(), updatedBlob.getBucket()); assertEquals(newMetadata, updatedBlob.getMetadata()); }
assertEquals(BUCKET, remoteBlob.getBucket()); assertTrue(blobSet.contains(remoteBlob.getName())); assertEquals(metadata, remoteBlob.getMetadata()); assertNull(remoteBlob.getContentType());
@Test public void testGetBlobAllSelectedFields() { String blobName = "test-get-all-selected-fields-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .setMetadata(ImmutableMap.of("k", "v")) .build(); assertNotNull(storage.create(blob)); Blob remoteBlob = storage.get(blob.getBlobId(), Storage.BlobGetOption.fields(BlobField.values())); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertEquals(ImmutableMap.of("k", "v"), remoteBlob.getMetadata()); assertNotNull(remoteBlob.getGeneratedId()); assertNotNull(remoteBlob.getSelfLink()); }
@Test public void testCopyBlob() { String sourceBlobName = "test-copy-blob-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v"); BlobInfo blob = BlobInfo.newBuilder(source).setContentType(CONTENT_TYPE).setMetadata(metadata).build(); Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); assertNotNull(remoteBlob); String targetBlobName = "test-copy-blob-target"; Storage.CopyRequest req = Storage.CopyRequest.of(source, BlobId.of(BUCKET, targetBlobName)); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); assertTrue(remoteBlob.delete()); assertTrue(storage.delete(BUCKET, targetBlobName)); }
@Test public void testCopyBlobNoContentType() { String sourceBlobName = "test-copy-blob-no-content-type-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); Blob remoteSourceBlob = storage.create(BlobInfo.newBuilder(source).build(), BLOB_BYTE_CONTENT); assertNotNull(remoteSourceBlob); String targetBlobName = "test-copy-blob-no-content-type-target"; ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v"); BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName).setMetadata(metadata).build(); Storage.CopyRequest req = Storage.CopyRequest.of(source, target); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertNull(copyWriter.getResult().getContentType()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); assertTrue(remoteSourceBlob.delete()); assertTrue(storage.delete(BUCKET, targetBlobName)); }
@Test public void testCopyBlobUpdateMetadata() { String sourceBlobName = "test-copy-blob-update-metadata-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); Blob remoteSourceBlob = storage.create(BlobInfo.newBuilder(source).build(), BLOB_BYTE_CONTENT); assertNotNull(remoteSourceBlob); String targetBlobName = "test-copy-blob-update-metadata-target"; ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v"); BlobInfo target = BlobInfo.newBuilder(BUCKET, targetBlobName) .setContentType(CONTENT_TYPE) .setMetadata(metadata) .build(); Storage.CopyRequest req = Storage.CopyRequest.of(source, target); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); assertTrue(remoteSourceBlob.delete()); assertTrue(storage.delete(BUCKET, targetBlobName)); }
assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); assertTrue(storage.delete(BUCKET, targetBlobName));
@Test public void testCopyBlobWithPredefinedAcl() { String sourceBlobName = "test-copy-blob-source"; BlobId source = BlobId.of(BUCKET, sourceBlobName); ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v"); BlobInfo blob = BlobInfo.newBuilder(source).setContentType(CONTENT_TYPE).setMetadata(metadata).build(); Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT); assertNotNull(remoteBlob); String targetBlobName = "test-copy-blob-target"; Storage.CopyRequest req = Storage.CopyRequest.newBuilder() .setSource(source) .setTarget( BlobId.of(BUCKET, targetBlobName), Storage.BlobTargetOption.predefinedAcl(Storage.PredefinedAcl.PUBLIC_READ)) .build(); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertNotNull(copyWriter.getResult().getAcl(User.ofAllUsers())); assertTrue(copyWriter.isDone()); assertTrue(remoteBlob.delete()); assertTrue(storage.delete(BUCKET, targetBlobName)); }
BLOB_BYTE_CONTENT, copyWriter.getResult().getContent(Blob.BlobSourceOption.decryptionKey(OTHER_BASE64_KEY))); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); req = assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); assertTrue(remoteBlob.delete());
assertEquals(MD5, blob.getMd5()); assertEquals(MEDIA_LINK, blob.getMediaLink()); assertEquals(METADATA, blob.getMetadata()); assertEquals(META_GENERATION, blob.getMetageneration()); assertEquals(OWNER, blob.getOwner()); assertNull(blob.getMd5()); assertNull(blob.getMediaLink()); assertNull(blob.getMetadata()); assertNull(blob.getMetageneration()); assertNull(blob.getOwner());
private Blob createMockBlobWithTableMetadata(Map<String, String> metadata) { Blob mockBlob = mock(Blob.class); Mockito.when(mockBlob.getMetadata()).thenReturn(metadata); return mockBlob; }
/** * Given a blob, return the {@link TableId} this blob should be inserted into. * @param blob the blob * @return the TableId this data should be loaded into, or null if we could not tell what * table it should be loaded into. */ public static TableId getTableFromBlob(Blob blob) { if (blob.getMetadata() == null || blob.getMetadata().get(GCSToBQWriter.GCS_METADATA_TABLE_KEY) == null) { logger.error("Found blob {}/{} with no metadata.", blob.getBucket(), blob.getName()); return null; } String serializedTableId = blob.getMetadata().get(GCSToBQWriter.GCS_METADATA_TABLE_KEY); Matcher matcher = METADATA_TABLE_PATTERN.matcher(serializedTableId); if (!matcher.find()) { logger.error("Found blob `{}/{}` with un-parsable table metadata.", blob.getBucket(), blob.getName()); return null; } String project = matcher.group("project"); String dataset = matcher.group("dataset"); String table = matcher.group("table"); logger.debug("Table data: project: {}; dataset: {}; table: {}", project, dataset, table); if (project == null) { return TableId.of(dataset, table); } else { return TableId.of(project, dataset, table); } }
@Test public void testGetTableFromBlobWithoutMetadata() { Blob mockBlob = mock(Blob.class); Mockito.when(mockBlob.getMetadata()).thenReturn(null); TableId tableId = GCSToBQLoadRunnable.getTableFromBlob(mockBlob); assertNull(tableId); }