/** Generate keys for each block and record them. * @throws IOException */ private void generateKeys(byte[][] dataBlocks, int offset) throws IOException { for(int i=0;i<dataBlocks.length;i++) { setKey(i + offset, encodeBlock(dataBlocks[i]).getClientKey()); } }
private void writeCheckBlock(int checkBlockNo, byte[] buf) throws IOException { parent.writeCheckBlock(segNo, checkBlockNo, buf); if(DEBUG_ENCODE) { SplitFileInserterSegmentStorage segment = segments[checkBlockNo + dataBlockCount]; ClientCHK key = segment.encodeBlock(buf).getClientKey(); segment.setKey(blockNumbers[checkBlockNo + dataBlockCount], key); } }
public static ClientCHK[] makeKeys(byte[][] blocks, byte[] cryptoKey, byte cryptoAlgorithm) throws CHKEncodeException { ClientCHK[] keys = new ClientCHK[blocks.length]; for(int i=0;i<blocks.length;i++) keys[i] = ClientCHKBlock.encodeSplitfileBlock(blocks[i], cryptoKey, cryptoAlgorithm).getClientKey(); return keys; }
private byte[][] readDataBlocks() throws IOException { RAFLock lock = parent.lockUnderlying(); try { byte[][] data = new byte[dataBlockCount][]; for(int i=0;i<dataBlockCount;i++) { data[i] = segments[i].readDataBlock(blockNumbers[i]); if(DEBUG_ENCODE) { ClientCHK key = segments[i].encodeBlock(data[i]).getClientKey(); segments[i].setKey(blockNumbers[i], key); } } return data; } finally { lock.unlock(); } }
/** Add a random block that has not been added already or decoded already. * @throws IOException */ private boolean addRandomBlock(SplitFileInserterStorage storage, SplitFileFetcherStorage fetcherStorage, Random random) throws IOException { int segCount = storage.segments.length; boolean[] exhaustedSegments = new boolean[segCount]; for(int i=0;i<segCount;i++) { while(true) { int segNo = random.nextInt(segCount); if(exhaustedSegments[segNo]) continue; SplitFileFetcherSegmentStorage segment = fetcherStorage.segments[segNo]; if(segment.isDecodingOrFinished()) { exhaustedSegments[segNo] = true; break; } while(true) { int blockNo = random.nextInt(segment.totalBlocks()); if(segment.hasBlock(blockNo)) { continue; } ClientCHKBlock block = storage.segments[segNo].encodeBlock(blockNo); boolean success = segment.onGotKey(block.getClientKey().getNodeCHK(), block.getBlock()); assertTrue(success); return true; } } } return false; }
private boolean checkEncodedDataBlocks(byte[][] checkBlocks, boolean[] checkBlocksPresent, SplitFileSegmentKeys keys, boolean capturingBinaryBlob) { for(int i=0;i<checkBlocks.length;i++) { if(checkBlocksPresent[i]) continue; ClientCHK decodeKey = keys.getKey(i+blocksForDecode(), null, false); // Encode it to check whether the key is the same. ClientCHKBlock block; try { block = ClientCHKBlock.encodeSplitfileBlock(checkBlocks[i], decodeKey.getCryptoKey(), decodeKey.getCryptoAlgorithm()); ClientCHK actualKey = block.getClientKey(); if(!actualKey.equals(decodeKey)) { Logger.error(this, "Splitfile check block "+i+" does not encode to expected key for "+this+" for "+parent); return false; } if(capturingBinaryBlob) parent.fetcher.maybeAddToBinaryBlob(block); } catch (CHKEncodeException e) { // Impossible! parent.fail(new FetchException(FetchExceptionMode.INTERNAL_ERROR, "Decoded block could not be encoded")); Logger.error(this, "Impossible: Decoded block could not be encoded"); return false; } } return true; }
private void checkBlock(byte[] data, boolean newAlgo) throws CHKEncodeException, InvalidCompressionCodecException, CHKVerifyException, CHKDecodeException, IOException { byte cryptoAlgorithm = newAlgo ? Key.ALGO_AES_CTR_256_SHA256 : Key.ALGO_AES_PCFB_256_SHA256; byte[] copyOfData = new byte[data.length]; System.arraycopy(data, 0, copyOfData, 0, data.length); ClientCHKBlock encodedBlock = ClientCHKBlock.encode(new ArrayBucket(data), false, false, (short)-1, data.length, null, false, null, cryptoAlgorithm); // Not modified in-place. assert(Arrays.equals(data, copyOfData)); ClientCHK key = encodedBlock.getClientKey(); if(newAlgo) { // Check with no JCA. ClientCHKBlock otherEncodedBlock = ClientCHKBlock.encode(new ArrayBucket(data), false, false, (short)-1, data.length, null, false, null, cryptoAlgorithm, true); assertTrue(key.equals(otherEncodedBlock.getClientKey())); assertTrue(Arrays.equals(otherEncodedBlock.getBlock().data, encodedBlock.getBlock().data)); assertTrue(Arrays.equals(otherEncodedBlock.getBlock().headers, encodedBlock.getBlock().headers)); } // Verify it. CHKBlock block = CHKBlock.construct(encodedBlock.getBlock().data, encodedBlock.getBlock().headers, cryptoAlgorithm); ClientCHKBlock checkBlock = new ClientCHKBlock(block, key); ArrayBucket checkData = (ArrayBucket) checkBlock.decode(new ArrayBucketFactory(), data.length, false); assert(Arrays.equals(checkData.toByteArray(), data)); if(newAlgo) { checkData = (ArrayBucket) checkBlock.decode(new ArrayBucketFactory(), data.length, false, true); assert(Arrays.equals(checkData.toByteArray(), data)); } }
public synchronized byte[] checkAndGetBlockData(int blockNum) throws IOException { if(!blockChooser.hasSucceeded(blockNum)) return null; ClientCHK key = getKey(blockNum); if(key == null) return null; for(int i=0;i<blocksFetched.length;i++) { if(blocksFetched[i] == blockNum) { byte[] buf = readBlock(i); try { ClientCHKBlock block = ClientCHKBlock.encodeSplitfileBlock(buf, key.getCryptoKey(), key.getCryptoAlgorithm()); if(!(block.getClientKey().equals(key))) { Logger.error(this, "Block "+blockNum+" in blocksFound["+i+"] is not valid!"); blockChooser.onUnSuccess(blockNum); succeeded = false; finished = false; } else { return buf; } } catch (CHKEncodeException e) { // Should not be possible. Logger.error(this, "Impossible: "+e); return null; } } } Logger.error(this, "Block "+blockNum+" in blocksFound but not in blocksFetched on "+this); return null; }
private void checkRAMStore(boolean newFormat) throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, newFormat); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); }
public void testDeletion() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException, InterruptedException { CHKStore store = new CHKStore(); SpeedyTicker st = new SpeedyTicker(); SlashdotStore<CHKBlock> ss = new SlashdotStore<>(store, 10, 0, 100, st, tbf); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test); store.put(block.getBlock(), false); // Do the same as what the ticker would have done... ss.purgeOldData(); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); if(verify == null) return; // Expected outcome String data = decodeBlock(verify, key); System.err.println("Got data: "+data+" but should have been deleted!"); fail(); }
private void checkBlocks(CHKStore store, boolean write, boolean expectFailure) throws CHKEncodeException, IOException, CHKVerifyException, CHKDecodeException { for(int i=0;i<5;i++) { // Encode a block String test = "test" + i; // Use new format for every other block to ensure they are mixed in the same store. ClientCHKBlock block = encodeBlock(test, (i & 1) == 1); if(write) store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); if(expectFailure) assertEquals(null, verify); else { String data = decodeBlock(verify, key); assertEquals(test, data); } } }
public void testSimpleCHK() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "testSaltedHashFreenetStoreCHK", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); saltStore.start(null, true); for(int i=0;i<5;i++) { String test = "test" + i; ClientCHKBlock block = encodeBlockCHK(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlockCHK(verify, key); assertEquals(test, data); } saltStore.close(); }
public void checkSaltedStore(boolean newFormat) throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "teststore", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); saltStore.start(null, true); for(int i=0;i<5;i++) { // Encode a block String test = "test" + i; ClientCHKBlock block = encodeBlock(test, newFormat); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); } saltStore.close(); }
public void testSimple() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); new SlashdotStore<CHKBlock>(store, 10, 30*1000, 5*1000, new TrivialTicker(exec), tbf); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); }
public void testMigrate() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); RAMFreenetStore<CHKBlock> ramStore = new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, true); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); CHKStore newStore = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(new File(tempDir, "saltstore"), "teststore", newStore, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); saltStore.start(null, true); ramStore.migrateTo(newStore, false); CHKBlock newVerify = store.fetch(key.getNodeCHK(), false, false, null); String newData = decodeBlock(newVerify, key); assertEquals(test, newData); saltStore.close(); }
public void testZeroSize() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "testCachingFreenetStoreCHK", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); CachingFreenetStoreTracker tracker = new CachingFreenetStoreTracker(0, cachingFreenetStorePeriod, ticker); CachingFreenetStore<CHKBlock> cachingStore = new CachingFreenetStore<CHKBlock>(store, saltStore, tracker); cachingStore.start(null, true); for(int i=0;i<5;i++) { String test = "test" + i; ClientCHKBlock block = encodeBlockCHK(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); // It should pass straight through. assertNotNull(saltStore.fetch(key.getRoutingKey(), key.getNodeCHK().getFullKey(), false, false, false, false, null)); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlockCHK(verify, key); assertEquals(test, data); } cachingStore.close(); }
public void testMigrateKeyed() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); RAMFreenetStore<CHKBlock> ramStore = new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, true); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); byte[] storeKey = new byte[32]; strongPRNG.nextBytes(storeKey); CHKStore newStore = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(new File(tempDir, "saltstore"), "teststore", newStore, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, storeKey); saltStore.start(null, true); ramStore.migrateTo(newStore, false); CHKBlock newVerify = store.fetch(key.getNodeCHK(), false, false, null); String newData = decodeBlock(newVerify, key); assertEquals(test, newData); saltStore.close(); }
public void testSimpleCHK() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { File f = new File(tempDir, "saltstore"); FileUtil.removeAll(f); CHKStore store = new CHKStore(); SaltedHashFreenetStore<CHKBlock> saltStore = SaltedHashFreenetStore.construct(f, "testCachingFreenetStoreCHK", store, weakPRNG, 10, false, SemiOrderedShutdownHook.get(), true, true, ticker, null); CachingFreenetStoreTracker tracker = new CachingFreenetStoreTracker(cachingFreenetStoreMaxSize, cachingFreenetStorePeriod, ticker); CachingFreenetStore<CHKBlock> cachingStore = new CachingFreenetStore<CHKBlock>(store, saltStore, tracker); cachingStore.start(null, true); for(int i=0;i<5;i++) { String test = "test" + i; ClientCHKBlock block = encodeBlockCHK(test); store.put(block.getBlock(), false); ClientCHK key = block.getClientKey(); // Check that it's in the cache, *not* the underlying store. assertEquals(saltStore.fetch(key.getRoutingKey(), key.getNodeCHK().getFullKey(), false, false, false, false, null), null); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlockCHK(verify, key); assertEquals(test, data); } cachingStore.close(); }
public void testSmallSplitfileCompletion() throws IOException, InsertException, MissingKeyException { Random r = new Random(12121); long size = 65536; // Exact multiple, so no last block LockableRandomAccessBuffer data = generateData(r, size); HashResult[] hashes = getHashes(data); MyCallback cb = new MyCallback(); InsertContext context = baseContext.clone(); KeysFetchingLocally keys = new MyKeysFetchingLocally(); SplitFileInserterStorage storage = new SplitFileInserterStorage(data, size, cb, null, new ClientMetadata(), false, null, smallRAFFactory, false, context, cryptoAlgorithm, cryptoKey, null, hashes, smallBucketFactory, checker, r, memoryLimitedJobRunner, jobRunner, ticker, keys, false, 0, 0, 0, 0); storage.start(); cb.waitForFinishedEncode(); assertEquals(storage.segments.length, 1); SplitFileInserterSegmentStorage segment = storage.segments[0]; assertEquals(segment.dataBlockCount, 2); assertEquals(segment.checkBlockCount, 3); assertEquals(segment.crossCheckBlockCount, 0); assertEquals(storage.getStatus(), Status.ENCODED); for(int i=0;i<segment.totalBlockCount;i++) { segment.onInsertedBlock(i, segment.encodeBlock(i).getClientKey()); } cb.waitForSucceededInsert(); assertEquals(storage.getStatus(), Status.SUCCEEDED); }
public void testRAMStoreOldBlocks() throws IOException, CHKEncodeException, CHKVerifyException, CHKDecodeException { CHKStore store = new CHKStore(); new RAMFreenetStore<CHKBlock>(store, 10); // Encode a block String test = "test"; ClientCHKBlock block = encodeBlock(test, false); store.put(block.getBlock(), true); ClientCHK key = block.getClientKey(); CHKBlock verify = store.fetch(key.getNodeCHK(), false, false, null); String data = decodeBlock(verify, key); assertEquals(test, data); // ignoreOldBlocks works. assertEquals(null, store.fetch(key.getNodeCHK(), false, true, null)); // Put it with oldBlock = false should unset the flag. store.put(block.getBlock(), false); verify = store.fetch(key.getNodeCHK(), false, true, null); data = decodeBlock(verify, key); assertEquals(test, data); }