@After @Override public void tearDown() throws Exception { FileUtils.deleteDirectory(new File(DataStoreUtils.getHomeDir())); super.tearDown(); }
public static void main(String... args) throws Exception { FileBlobStore store = new FileBlobStore("target/temp"); String id = addFiles(store, "~/temp/ds"); extractFiles(store, id, "target/test"); }
@Test public void uploadCallback() throws Exception { assumeTrue(supportsStatsCollection()); TestCollector collector = new TestCollector(); setupCollector(collector); int size = 10 * 1024; store.writeBlob(randomStream(42, size)); //For chunked storage the actual stored size is greater than the file size assertCollectedSize(collector.size, size); assertEquals(1, collector.uploadCount); }
private Set<String> createArtifacts() throws Exception { Set<String> ids = Sets.newHashSet(); int number = 10; for (int i = 0; i < number; i++) { String id = store.writeBlob(randomStream(i, getArtifactSize())); Iterator<String> iter = store.resolveChunks(id.toString()); while (iter.hasNext()) { ids.add(iter.next()); } } return ids; }
@Test public void list() throws Exception { Set<String> ids = createArtifacts(); Iterator<String> iter = store.getAllChunkIds(0); while (iter.hasNext()) { ids.remove(iter.next()); } assertTrue("unexpected ids in store: " + ids, ids.isEmpty()); }
@Test public void testSmall() throws Exception { doTest(10, 300); }
private void doTest(int maxLength, int count) throws Exception { String[] s = new String[count * 2]; Random r = new Random(0); for (int i = 0; i < s.length;) { byte[] data = new byte[r.nextInt(maxLength)]; r.nextBytes(data); s[i++] = store.writeBlob(new ByteArrayInputStream(data)); s[i++] = store.writeBlob(new ByteArrayInputStream(data)); } r.setSeed(0); for (int i = 0; i < s.length;) { int expectedLen = r.nextInt(maxLength); byte[] expectedData = new byte[expectedLen]; r.nextBytes(expectedData); assertEquals(expectedLen, store.getBlobLength(s[i++])); String id = s[i++]; doTestRead(expectedData, expectedLen, id); } }
private void doTestRead(byte[] expectedData, int expectedLen, String id) throws Exception { byte[] got = readFully(id); assertEquals(expectedLen, got.length); assertEquals(expectedData.length, got.length); for (int i = 0; i < got.length; i++) { assertEquals(expectedData[i], got[i]); } }
private static void addFiles(ArrayList<String> list, File file) { if (file.isDirectory()) { for (File f : file.listFiles()) { addFiles(list, f); } return; } if (!file.isFile()) { return; } list.add(file.getAbsolutePath()); }
@Test public void deleteUpdatedBlob() throws Exception { String id = store.writeBlob(randomStream(0, getArtifactSize())); Thread.sleep(100); long beforeUpdateTime = System.currentTimeMillis(); Thread.sleep(1000); // Should update the timestamp String id2 = store.writeBlob(randomStream(0, getArtifactSize())); assertEquals(id, id2); Set<String> chunks = Sets.newHashSet(); Iterator<String> iter = store.resolveChunks(id.toString()); while (iter.hasNext()) { chunks.add(iter.next()); } long count = store.countDeleteChunks(Lists.newArrayList(chunks), beforeUpdateTime); assertEquals("Deleted updated blobs", 0, count); }
@Test public void delete() throws Exception { Set<String> ids = createArtifacts(); store.deleteChunks(Lists.newArrayList(ids), 0); Iterator<String> iter = store.getAllChunkIds(0); Set<String> ret = Sets.newHashSet(); while (iter.hasNext()) { ret.add(iter.next()); } assertTrue(ret.toString(), ret.isEmpty()); }
@Test public void testMedium() throws Exception { doTest(100, 100); }
@Test public void testCombinedIdentifier() throws Exception { String id = store.writeBlob(new ByteArrayInputStream(new byte[2])); assertEquals(2, store.getBlobLength(id)); String combinedId = id + id; assertEquals(4, store.getBlobLength(combinedId)); doTestRead(new byte[4], 4, combinedId); }
public static String addFiles(BlobStore store, String dir) throws Exception { ArrayList<String> list = new ArrayList<String>(); String root = new File(dir).getAbsolutePath(); String parent = new File(dir).getParentFile().getAbsolutePath(); addFiles(list, new File(root)); JsopBuilder listing = new JsopBuilder(); listing.object(); for (String f : list) { FileInputStream in = new FileInputStream(f); String id = store.writeBlob(in); in.close(); String name = f.substring(parent.length()); listing.key(name).value(id); listing.newline(); } listing.endObject(); String l = listing.toString(); String id = store.writeBlob(new ByteArrayInputStream(l.getBytes("UTF-8"))); return id; }
@Test public void downloadCallback() throws Exception { assumeTrue(supportsStatsCollection()); TestCollector collector = new TestCollector(); setupCollector(collector); int size = 10 * 1024; String id = store.writeBlob(randomStream(42, size)); store.clearCache(); collector.reset(); InputStream is = store.getInputStream(id); CountingOutputStream cos = new CountingOutputStream(new NullOutputStream()); IOUtils.copy(is, cos); is.close(); assertEquals(size, cos.getCount()); //For chunked storage the actual stored size is greater than the file size assertCollectedSize(collector.size, size); assertEquals(1, collector.downloadCount); }
@Override @After public void tearDown() throws Exception { ((CloudBlobStore) store).deleteBucket(); super.tearDown(); }
@Test public void deleteCount() throws Exception { Set<String> ids = createArtifacts(); long count = store.countDeleteChunks(Lists.newArrayList(ids), 0); Iterator<String> iter = store.getAllChunkIds(0); Set<String> ret = Sets.newHashSet(); while (iter.hasNext()) { ret.add(iter.next()); } assertTrue(ret.toString(), ret.isEmpty()); assertEquals(ids.size(), count); }
@Test public void testLarge() throws Exception { doTest(1000, 10); }
@After @Override public void tearDown() throws Exception { super.tearDown(); if (blobStore != null) { empty(blobStore); blobStore.close(); } }