@Override public Collection<Block> getBySequenceHash(ByteArray hash) { return mem.getBySequenceHash(hash); }
private static void populateIndex(CloneIndex duplicationIndex, Collection<Block> duplicationBlocks) { for (Block block : duplicationBlocks) { duplicationIndex.insert(block); } }
public Collection<Block> getByInputFile(String resourceKey) { return mem.getByResourceId(resourceKey); }
/** * Given: index with initial capacity 1. * Expected: size and capacity should be increased after insertion of two blocks. */ @Test public void should_increase_capacity() { CloneIndex index = new PackedMemoryCloneIndex(8, 1); index.insert(newBlock("a", 1)); index.insert(newBlock("a", 2)); assertThat(index.getByResourceId("a").size(), is(2)); }
@Override public Iterator<ResourceBlocks> iterator() { return mem.iterator(); }
@Override public int noResources() { return mem.noResources(); }
@Override public Iterator<ResourceBlocks> iterator() { return mem.iterator(); }
@Override public int noResources() { return mem.noResources(); }
protected static CloneIndex createIndex(Block[]... blocks) { CloneIndex cloneIndex = new MemoryCloneIndex(); for (Block[] b : blocks) { for (Block block : b) { cloneIndex.insert(block); } } return cloneIndex; }
private static Map<String, List<Block>> retrieveFromIndex(CloneIndex index, String originResourceId, Set<ByteArray> hashes) { Map<String, List<Block>> collection = new HashMap<>(); for (ByteArray hash : hashes) { Collection<Block> blocks = index.getBySequenceHash(hash); for (Block blockFromIndex : blocks) { // Godin: skip blocks for this file if they come from index String resourceId = blockFromIndex.getResourceId(); if (!originResourceId.equals(resourceId)) { List<Block> list = collection.get(resourceId); if (list == null) { list = new ArrayList<>(); collection.put(resourceId, list); } list.add(blockFromIndex); } } } return collection; }
private List<CloneGroup> detect(File file) { Collection<Block> fileBlocks = index.getByResourceId(file.getAbsolutePath()); return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); }
@Override public Iterator<ResourceBlocks> iterator() { return mem.iterator(); }
private void addToIndex(File file) throws IOException { List<Block> blocks = bridge.chunk(file.getAbsolutePath(), file.getAbsolutePath(), Files.newBufferedReader(file.toPath(), StandardCharsets.UTF_8)); for (Block block : blocks) { index.insert(block); } }
ByteArray hash = entry.getKey(); BlocksGroup group = entry.getValue(); for (Block blockFromIndex : cloneIndex.getBySequenceHash(hash)) {
public Collection<Block> getByInputFile(String resourceKey) { return mem.getByResourceId(resourceKey); }
/** * Given: index, which accepts blocks with 4-byte hash. * Expected: exception during insertion of block with 8-byte hash. */ @Test(expected = IllegalArgumentException.class) public void attempt_to_insert_hash_of_incorrect_size() { CloneIndex index = new PackedMemoryCloneIndex(4, 1); index.insert(newBlock("a", 1)); }
/** * Given: file with repeated hashes * Expected: only one query of index for each unique hash */ @Test public void only_one_query_of_index_for_each_unique_hash() { CloneIndex index = spy(createIndex()); Block[] fileBlocks = newBlocks("a", "1 2 1 2"); detect(index, fileBlocks); verify(index).getBySequenceHash(new ByteArray("01")); verify(index).getBySequenceHash(new ByteArray("02")); verifyNoMoreInteractions(index); }
public Collection<Block> getByInputFile(String resourceKey) { return mem.getByResourceId(resourceKey); }
public void insert(InputFile inputFile, Collection<Block> blocks) { if (settings.isCrossProjectDuplicationEnabled()) { int id = ((DefaultInputFile) inputFile).scannerId(); if (publisher.getWriter().hasComponentData(FileStructure.Domain.CPD_TEXT_BLOCKS, id)) { throw new UnsupportedOperationException("Trying to save CPD tokens twice for the same file is not supported: " + inputFile.absolutePath()); } final ScannerReport.CpdTextBlock.Builder builder = ScannerReport.CpdTextBlock.newBuilder(); publisher.getWriter().writeCpdTextBlocks(id, blocks.stream().map(block -> { builder.clear(); builder.setStartLine(block.getStartLine()); builder.setEndLine(block.getEndLine()); builder.setStartTokenIndex(block.getStartUnit()); builder.setEndTokenIndex(block.getEndUnit()); builder.setHash(block.getBlockHash().toHexString()); return builder.build(); }).collect(Collectors.toList())); } for (Block block : blocks) { mem.insert(block); } if (blocks.isEmpty()) { LOG.debug("Not enough content in '{}' to have CPD blocks, it will not be part of the duplication detection", inputFile.relativePath()); } indexedFiles.add(inputFile); }
/** * Given: index, which accepts blocks with 4-byte hash. * Expected: exception during search by 8-byte hash. */ @Test(expected = IllegalArgumentException.class) public void attempt_to_find_hash_of_incorrect_size() { CloneIndex index = new PackedMemoryCloneIndex(4, 1); index.getBySequenceHash(new ByteArray(1L)); }