private static List<CloneGroup> detect(CloneIndex index, Collection<Block> blocks) { return SuffixTreeCloneDetectionAlgorithm.detect(index, blocks); }
private static TextSet createTextSet(CloneIndex index, Collection<Block> fileBlocks) { Set<ByteArray> hashes = new HashSet<>(); for (Block fileBlock : fileBlocks) { hashes.add(fileBlock.getBlockHash()); } String originResourceId = fileBlocks.iterator().next().getResourceId(); Map<String, List<Block>> fromIndex = retrieveFromIndex(index, originResourceId, hashes); if (fromIndex.isEmpty() && hashes.size() == fileBlocks.size()) { // optimization for the case when there is no duplications return null; } return createTextSet(fileBlocks, fromIndex); }
public static List<CloneGroup> detect(CloneIndex cloneIndex, Collection<Block> fileBlocks) { if (fileBlocks.isEmpty()) { return Collections.emptyList(); } TextSet text = createTextSet(cloneIndex, fileBlocks); if (text == null) { return Collections.emptyList(); } DuplicationsCollector reporter = new DuplicationsCollector(text); Search.perform(text, reporter); return reporter.getResult(); }
public static List<CloneGroup> detect(CloneIndex cloneIndex, Collection<Block> fileBlocks) { if (fileBlocks.isEmpty()) { return Collections.EMPTY_LIST; } TextSet text = createTextSet(cloneIndex, fileBlocks); if (text == null) { return Collections.EMPTY_LIST; } DuplicationsCollector reporter = new DuplicationsCollector(text); Search.perform(text, reporter); return reporter.getResult(); }
@Override protected List<CloneGroup> detect(CloneIndex index, Block[] fileBlocks) { return SuffixTreeCloneDetectionAlgorithm.detect(index, Arrays.asList(fileBlocks)); }
private static TextSet createTextSet(CloneIndex index, Collection<Block> fileBlocks) { Set<ByteArray> hashes = Sets.newHashSet(); for (Block fileBlock : fileBlocks) { hashes.add(fileBlock.getBlockHash()); } String originResourceId = fileBlocks.iterator().next().getResourceId(); Map<String, List<Block>> fromIndex = retrieveFromIndex(index, originResourceId, hashes); if (fromIndex.isEmpty() && hashes.size() == fileBlocks.size()) { // optimization for the case when there is no duplications return null; } return createTextSet(fileBlocks, fromIndex); }
private List<CloneGroup> detect(File file) { Collection<Block> fileBlocks = index.getByResourceId(file.getAbsolutePath()); return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); }
public void computeCpd(Component component, Collection<Block> originBlocks, Collection<Block> duplicationBlocks) { CloneIndex duplicationIndex = new PackedMemoryCloneIndex(); populateIndex(duplicationIndex, originBlocks); populateIndex(duplicationIndex, duplicationBlocks); List<CloneGroup> duplications = SuffixTreeCloneDetectionAlgorithm.detect(duplicationIndex, originBlocks); Iterable<CloneGroup> filtered = from(duplications).filter(getNumberOfUnitsNotLessThan(component.getFileAttributes().getLanguageKey())); addDuplications(component, filtered); }
@VisibleForTesting void runCpdAnalysis(ExecutorService executorService, DefaultInputFile inputFile, Collection<Block> fileBlocks, long timeout) { LOG.debug("Detection of duplications for {}", inputFile.absolutePath()); progressReport.message(String.format("%d/%d - current file: %s", count, total, inputFile.absolutePath())); List<CloneGroup> duplications; Future<List<CloneGroup>> futureResult = executorService.submit(() -> SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks)); try { duplications = futureResult.get(timeout, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { LOG.warn("Timeout during detection of duplications for {}", inputFile.absolutePath()); futureResult.cancel(true); return; } catch (Exception e) { throw new IllegalStateException("Fail during detection of duplication for " + inputFile.absolutePath(), e); } List<CloneGroup> filtered; if (!"java".equalsIgnoreCase(inputFile.language())) { int minTokens = settings.getMinimumTokens(inputFile.language()); Predicate<CloneGroup> minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(minTokens); filtered = duplications.stream() .filter(minimumTokensPredicate) .collect(Collectors.toList()); } else { filtered = duplications; } saveDuplications(inputFile, filtered); }
@Override public List<CloneGroup> call() { return SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); } }
public void computeCpd(Component component, Collection<Block> originBlocks, Collection<Block> duplicationBlocks) { CloneIndex duplicationIndex = new PackedMemoryCloneIndex(); populateIndex(duplicationIndex, originBlocks); populateIndex(duplicationIndex, duplicationBlocks); List<CloneGroup> duplications = SuffixTreeCloneDetectionAlgorithm.detect(duplicationIndex, originBlocks); Iterable<CloneGroup> filtered = from(duplications).filter(getNumberOfUnitsNotLessThan(component.getFileAttributes().getLanguageKey())); addDuplications(component, filtered); }
@VisibleForTesting void runCpdAnalysis(ExecutorService executorService, DefaultInputFile inputFile, Collection<Block> fileBlocks, long timeout) { LOG.debug("Detection of duplications for {}", inputFile.absolutePath()); progressReport.message(String.format("%d/%d - current file: %s", count, total, inputFile.absolutePath())); List<CloneGroup> duplications; Future<List<CloneGroup>> futureResult = executorService.submit(() -> SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks)); try { duplications = futureResult.get(timeout, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { LOG.warn("Timeout during detection of duplications for {}", inputFile.absolutePath()); futureResult.cancel(true); return; } catch (Exception e) { throw new IllegalStateException("Fail during detection of duplication for " + inputFile.absolutePath(), e); } List<CloneGroup> filtered; if (!"java".equalsIgnoreCase(inputFile.language())) { int minTokens = settings.getMinimumTokens(inputFile.language()); Predicate<CloneGroup> minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(minTokens); filtered = duplications.stream() .filter(minimumTokensPredicate) .collect(Collectors.toList()); } else { filtered = duplications; } saveDuplications(inputFile, filtered); }
private void runCpdAnalysis(String resource, Collection<Block> fileBlocks) { LOG.debug("Detection of duplications for {}", resource); BatchComponent component = batchComponentCache.get(resource); if (component == null) { LOG.error("Resource not found in component cache: {}. Skipping CPD computation for it", resource); return; } List<CloneGroup> duplications; try { duplications = SuffixTreeCloneDetectionAlgorithm.detect(index, fileBlocks); } catch (Exception e) { throw new IllegalStateException("Fail during detection of duplication for " + resource, e); } InputFile inputFile = (InputFile) component.inputComponent(); List<CloneGroup> filtered; if (!"java".equalsIgnoreCase(inputFile.language())) { Predicate<CloneGroup> minimumTokensPredicate = DuplicationPredicates.numberOfUnitsNotLessThan(getMinimumTokens(inputFile.language())); filtered = from(duplications).filter(minimumTokensPredicate).toList(); } else { filtered = duplications; } saveDuplications(component, filtered); }