@Override public Iterator<String> getAllChunkIds(long maxLastModifiedTime) throws Exception { return delegate.getAllChunkIds(maxLastModifiedTime); }
@Override public Iterator<String> resolveChunks(String blobId) throws IOException { return delegate.resolveChunks(blobId); }
@Override public long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception { long numDeleted = delegate.countDeleteChunks(chunkIds, maxLastModifiedTime); numChunks -= numDeleted; return numDeleted; }
@Override public String writeBlob(InputStream in) throws IOException { String blobId = delegate.writeBlob(in); numChunks += Iterators.size(delegate.resolveChunks(blobId)); return blobId; }
List<String> chunkIds = Lists.newArrayList(blobStore.resolveChunks(deletedBlobId)); if (chunkIds.size() > 0) { long deleted = blobStore.countDeleteChunks(chunkIds, 0); if (deleted < 1) { LOG.warn("Blob {} in file {} not deleted", deletedBlobId, deletedBlobListFile);
@Test public void deleteUpdatedBlob() throws Exception { String id = store.writeBlob(randomStream(0, getArtifactSize())); Thread.sleep(100); long beforeUpdateTime = System.currentTimeMillis(); Thread.sleep(1000); // Should update the timestamp String id2 = store.writeBlob(randomStream(0, getArtifactSize())); assertEquals(id, id2); Set<String> chunks = Sets.newHashSet(); Iterator<String> iter = store.resolveChunks(id.toString()); while (iter.hasNext()) { chunks.add(iter.next()); } long count = store.countDeleteChunks(Lists.newArrayList(chunks), beforeUpdateTime); assertEquals("Deleted updated blobs", 0, count); }
@Test public void deleteCount() throws Exception { Set<String> ids = createArtifacts(); long count = store.countDeleteChunks(Lists.newArrayList(ids), 0); Iterator<String> iter = store.getAllChunkIds(0); Set<String> ret = Sets.newHashSet(); while (iter.hasNext()) { ret.add(iter.next()); } assertTrue(ret.toString(), ret.isEmpty()); assertEquals(ids.size(), count); }
byte[] data = new byte[i]; String id; id = store.writeBlob(new ByteArrayInputStream(data)); data = new byte[i]; Arrays.fill(data, (byte) 1); id = store.writeBlob(new ByteArrayInputStream(data)); store.startMark(); store.sweep(); for (String id : map.keySet()) { byte[] test = readFully(id); store.clearInUse(); store.startMark(); for (String id : map.keySet()) { byte[] d = map.get(id); store.getBlobLength(id); count = store.sweep(); store.clearInUse(); store.clearCache(); store.startMark(); count = store.sweep(); assertTrue("count: " + count, count > 0); int failedCount = 0;
@Test public void delete() throws Exception { Set<String> ids = createArtifacts(); store.deleteChunks(Lists.newArrayList(ids), 0); Iterator<String> iter = store.getAllChunkIds(0); Set<String> ret = Sets.newHashSet(); while (iter.hasNext()) { ret.add(iter.next()); } assertTrue(ret.toString(), ret.isEmpty()); }
@Test public void testWriteFile() throws Exception { store.setBlockSize(1024 * 1024); byte[] data = new byte[4 * 1024 * 1024]; Random r = new Random(0); r.nextBytes(data); String tempFileName = "target/temp/test"; File tempFile = new File(tempFileName); tempFile.getParentFile().mkdirs(); OutputStream out = new FileOutputStream(tempFile, false); out.write(data); out.close(); String s = store.writeBlob(tempFileName); assertEquals(data.length, store.getBlobLength(s)); byte[] buff = new byte[1]; for (int i = 0; i < data.length; i += 1024) { store.readBlob(s, i, buff, 0, 1); assertEquals(data[i], buff[0]); } try { store.writeBlob(tempFileName + "_wrong"); fail(); } catch (Exception e) { // expected } }
@Test public void testReference() throws Exception { assumeThat(store, instanceOf(AbstractBlobStore.class)); AbstractBlobStore abs = (AbstractBlobStore) store; Random r = new Random(); byte[] key = new byte[256]; r.nextBytes(key); abs.setReferenceKey(key); byte[] data = new byte[1000]; r.nextBytes(data); String blobId = store.writeBlob(new ByteArrayInputStream(data)); String reference = store.getReference(blobId); String blobId2 = store.getBlobId(reference); assertEquals(blobId, blobId2); }
@Test public void downloadCallback() throws Exception { assumeTrue(supportsStatsCollection()); TestCollector collector = new TestCollector(); setupCollector(collector); int size = 10 * 1024; String id = store.writeBlob(randomStream(42, size)); store.clearCache(); collector.reset(); InputStream is = store.getInputStream(id); CountingOutputStream cos = new CountingOutputStream(new NullOutputStream()); IOUtils.copy(is, cos); is.close(); assertEquals(size, cos.getCount()); //For chunked storage the actual stored size is greater than the file size assertCollectedSize(collector.size, size); assertEquals(1, collector.downloadCount); }
@Test public void testCombinedIdentifier() throws Exception { String id = store.writeBlob(new ByteArrayInputStream(new byte[2])); assertEquals(2, store.getBlobLength(id)); String combinedId = id + id; assertEquals(4, store.getBlobLength(combinedId)); doTestRead(new byte[4], 4, combinedId); }
@Test public void testCloseStream() throws Exception { final AtomicBoolean closed = new AtomicBoolean(); InputStream in = new InputStream() { @Override public void close() { closed.set(true); } @Override public int read() throws IOException { return -1; } }; store.writeBlob(in); assertTrue(closed.get()); }
@Override public void clearCache() { delegate.clearCache(); }
@Override public void clearInUse() { delegate.clearInUse(); }
@Override @Deprecated public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception { numChunks -= chunkIds.size(); return delegate.deleteChunks(chunkIds, maxLastModifiedTime); }
@Override public String writeBlob(InputStream in, BlobOptions options) throws IOException { String blobId = delegate.writeBlob(in, options); numChunks += Iterators.size(delegate.resolveChunks(blobId)); return blobId; }
List<String> chunkIds = Lists.newArrayList(blobStore.resolveChunks(deletedBlobId)); if (chunkIds.size() > 0) { long deleted = blobStore.countDeleteChunks(chunkIds, 0); if (deleted < 1) { LOG.warn("Blob {} in file {} not deleted", deletedBlobId, deletedBlobListFile);
private void doTest(int maxLength, int count) throws Exception { String[] s = new String[count * 2]; Random r = new Random(0); for (int i = 0; i < s.length;) { byte[] data = new byte[r.nextInt(maxLength)]; r.nextBytes(data); s[i++] = store.writeBlob(new ByteArrayInputStream(data)); s[i++] = store.writeBlob(new ByteArrayInputStream(data)); } r.setSeed(0); for (int i = 0; i < s.length;) { int expectedLen = r.nextInt(maxLength); byte[] expectedData = new byte[expectedLen]; r.nextBytes(expectedData); assertEquals(expectedLen, store.getBlobLength(s[i++])); String id = s[i++]; doTestRead(expectedData, expectedLen, id); } }