@Override public Key getNodeKey(boolean cloneKey) { return cloneKey ? getNodeCHK().cloneKey() : getNodeCHK(); }
/** * Construct from data retrieved, and a key. * Do not do full decode. Verify what can be verified without doing * a full decode. * @param key2 The client key. * @param header The header. * @param data The data. */ public ClientCHKBlock(byte[] data, byte[] header, ClientCHK key2, boolean verify) throws CHKVerifyException { block = new CHKBlock(data, header, key2.getNodeCHK(), verify, key2.cryptoAlgorithm); this.key = key2; }
private boolean keysValid(ClientCHK[] keys) { for(ClientCHK key: keys) if(key.getNodeCHK().getRoutingKey() == null) return false; return true; }
public NodeCHK getCHK(int block) { if(block < dataBlocks.length) return dataKeys[block].getNodeCHK(); else return checkKeys[block-dataBlocks.length].getNodeCHK(); }
private ClientKeyBlock fetch(ClientCHK clientCHK, boolean canReadClientCache, boolean canWriteClientCache, boolean canWriteDatastore) throws CHKVerifyException { CHKBlock block = fetch(clientCHK.getNodeCHK(), false, canReadClientCache, canWriteClientCache, canWriteDatastore, false, null); if(block == null) return null; return new ClientCHKBlock(block, clientCHK); }
@Override public boolean run(ClientContext context) { try { // FIXME CPU USAGE Add another API to the segment to avoid re-decoding. SplitFileSegmentKeys keys = segments[blockNo].getSegmentKeys(); if(keys == null) return false; boolean success = segments[blockNo].innerOnGotKey(key.getNodeCHK(), block, keys, blockNumbers[blockNo], data); if(success) { if(logMINOR) Logger.minor(this, "Successfully decoded cross-segment block"); } else { // Not really a big deal, but potentially interesting... Logger.warning(this, "Decoded cross-segment block but not wanted by segment"); } } catch (IOException e) { parent.failOnDiskError(e); return true; } return false; } });
private void checkDecodedBlock(int i, byte[] data) { ClientCHK key = getKey(i); if(key == null) { Logger.error(this, "Key not found"); failOffThread(new FetchException(FetchExceptionMode.INTERNAL_ERROR, "Key not found")); return; } ClientCHKBlock block = encodeBlock(key, data); String decoded = i >= dataBlockCount ? "Encoded" : "Decoded"; if(block == null || !key.getNodeCHK().equals(block.getKey())) { Logger.error(this, decoded+" cross-segment block "+i+" failed!"); failOffThread(new FetchException(FetchExceptionMode.SPLITFILE_DECODE_ERROR, decoded+" cross-segment block does not match expected key")); return; } else { reportBlockToSegmentOffThread(i, key, block, data); } }
public void testRAMStoreOldBlocks() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, false); store.put(block.getBlock(), true); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); // ignoreOldBlocks works. assertEquals(null, store.fetch(key.getNodeCHK(), false, true, null)); // Put it with oldBlock = false should unset the flag. store.put(block.getBlock(), false); verify = store.fetch(key.getNodeCHK(), false, true, null); data = decodeBlock(verify, key); assertEquals(test, data); }
public void testMigrate() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); RAMFreenetStore<CHKBlock> ramStore = new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, true); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); CHKStore newStore = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(new File(tempDir, "saltstore"), "teststore", newStore, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); saltStore.start(null, true); ramStore.migrateTo(newStore, false); CHKBlock newVerify = store.fetch(key.getNodeCHK(), false, false, null); String newData = decodeBlock(newVerify, key); assertEquals(test, newData); saltStore.close(); }
private void checkRAMStore(boolean newFormat) throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, newFormat); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); }
public void testDeletion() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException, InterruptedException { CHKStore store = new CHKStore(); SpeedyTicker st = new SpeedyTicker(); SlashdotStore<CHKBlock> ss = new SlashdotStore<>(store, 10, 0, 100, st, tbf); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test); store.put(block.getBlock(), false); // Do the same as what the ticker would have done... ss.purgeOldData(); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); if(verify == null) return; // Expected outcome String data = decodeBlock(verify, key); System.err.println("Got data: "+data+" but should have been deleted!"); fail(); }
private void checkBlocks(CHKStore store, boolean write, boolean expectFailure) throws CHKEncodeException, IOException, CHKVerifyException, CHKDecodeException { for(int i=0;i<5;i++) { // Encode a block String test = "test" + i; // Use new format for every other block to ensure they are mixed in the same store. ClientCHKBlock block = encodeBlock(test, (i & 1) == 1); if(write) store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); if(expectFailure) assertEquals(null, verify); else { String data = decodeBlock(verify, key); assertEquals(test, data); } } }
public void testSimpleCHK() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "testCachingFreenetStoreCHK", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); CachingFreenetStoreTracker tracker = new CachingFreenetStoreTracker(cachingFreenetStoreMaxSize, cachingFreenetStorePeriod, ticker); CachingFreenetStore<CHKBlock> cachingStore = new CachingFreenetStore<CHKBlock>(store, saltStore, tracker); cachingStore.start(null, true); for(int i=0;i<5;i++) { String test = "test" + i; ClientCHKBlock block = encodeBlockCHK(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); // Check that it's in the cache, *not* the underlying store. assertEquals(saltStore.fetch(key.getRoutingKey(), key.getNodeCHK().getFullKey(), false, false, false, false, null), null); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlockCHK(verify, key); assertEquals(test, data); } cachingStore.close(); }
public void testZeroSize() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "testCachingFreenetStoreCHK", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); CachingFreenetStoreTracker tracker = new CachingFreenetStoreTracker(0, cachingFreenetStorePeriod, ticker); CachingFreenetStore<CHKBlock> cachingStore = new CachingFreenetStore<CHKBlock>(store, saltStore, tracker); cachingStore.start(null, true); for(int i=0;i<5;i++) { String test = "test" + i; ClientCHKBlock block = encodeBlockCHK(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); // It should pass straight through. assertNotNull(saltStore.fetch(key.getRoutingKey(), key.getNodeCHK().getFullKey(), false, false, false, false, null)); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlockCHK(verify, key); assertEquals(test, data); } cachingStore.close(); }
public void testMigrateKeyed() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); RAMFreenetStore<CHKBlock> ramStore = new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, true); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); byte[] storeKey = new byte[32]; strongPRNG.nextBytes(storeKey); CHKStore newStore = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(new File(tempDir, "saltstore"), "teststore", newStore, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, storeKey); saltStore.start(null, true); ramStore.migrateTo(newStore, false); CHKBlock newVerify = store.fetch(key.getNodeCHK(), false, false, null); String newData = decodeBlock(newVerify, key); assertEquals(test, newData); saltStore.close(); }
public void checkSaltedStore(boolean newFormat) throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "teststore", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); saltStore.start(null, true); for(int i=0;i<5;i++) { // Encode a block String test = "test" + i; ClientCHKBlock block = encodeBlock(test, newFormat); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); } saltStore.close(); }
public void testSimpleCHK() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "testSaltedHashFreenetStoreCHK", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); saltStore.start(null, true); for(int i=0;i<5;i++) { String test = "test" + i; ClientCHKBlock block = encodeBlockCHK(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlockCHK(verify, key); assertEquals(test, data); } saltStore.close(); }
public void testSimple() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); new SlashdotStore<CHKBlock>(store, 10, 30*1000, 5*1000, new TrivialTicker(exec), tbf); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); }
private void testDataBlocksOnly(TestSplitfile test) throws IOException, CHKEncodeException, FetchException, MetadataParseException { StorageCallback cb = test.createStorageCallback(); SplitFileFetcherStorage storage = test.createStorage(cb); SplitFileFetcherSegmentStorage segment = storage.segments[0]; for(int i=0;i<test.checkBlocks.length;i++) { segment.onNonFatalFailure(test.dataBlocks.length+i); } for(int i=0;i<test.dataBlocks.length;i++) { assertFalse(segment.hasStartedDecode()); assertTrue(segment.onGotKey(test.dataKeys[i].getNodeCHK(), test.encodeDataBlock(i))); cb.markDownloadedBlock(i); } cb.checkFailed(); assertTrue(segment.hasStartedDecode()); cb.checkFailed(); waitForDecode(segment); cb.checkFailed(); cb.waitForFinished(); cb.checkFailed(); test.verifyOutput(storage); cb.checkFailed(); storage.finishedFetcher(); cb.checkFailed(); waitForFinished(segment); cb.checkFailed(); cb.waitForFree(storage); cb.checkFailed(); }
assertTrue(segment.onGotKey(test.checkKeys[i].getNodeCHK(), test.encodeCheckBlock(i))); cb.markDownloadedBlock(i + test.dataBlocks.length);