private List<Statement> chunk(File file) { Reader reader = null; try { reader = new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8); return statementChunker.chunk(tokenChunker.chunk(reader)); } catch (FileNotFoundException e) { throw new RuntimeException(e); } finally { IOUtils.closeQuietly(reader); } }
private List<Statement> chunk(String sourceCode) { return statementChunker.chunk(tokenChunker.chunk(sourceCode)); }
private List<CloneGroup> detect(String... lines) { String sourceCode = asList(lines).stream().collect(joining("\n")); MemoryCloneIndex index = new MemoryCloneIndex(); List<Statement> statements = STATEMENT_CHUNKER.chunk(TOKEN_CHUNKER.chunk(sourceCode)); List<Block> blocks = BLOCK_CHUNKER.chunk("resourceId", statements); for (Block block : blocks) { index.insert(block); } return detect(index, blocks); }
private void createIndex(Iterable<InputFile> sourceFiles) { TokenChunker tokenChunker = JavaTokenProducer.build(); StatementChunker statementChunker = JavaStatementBuilder.build(); BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE); for (InputFile inputFile : sourceFiles) { LOG.debug("Populating index from {}", inputFile); String resourceEffectiveKey = inputFile.key(); List<Statement> statements; try (InputStream is = inputFile.inputStream(); Reader reader = new InputStreamReader(is, inputFile.charset())) { statements = statementChunker.chunk(tokenChunker.chunk(reader)); } catch (FileNotFoundException e) { throw new IllegalStateException("Cannot find file " + inputFile.file(), e); } catch (IOException e) { throw new IllegalStateException("Exception handling file: " + inputFile.file(), e); } List<Block> blocks; try { blocks = blockChunker.chunk(resourceEffectiveKey, statements); } catch (Exception e) { throw new IllegalStateException("Cannot process file " + inputFile.file(), e); } index.insert(inputFile, blocks); } }
@Test(expected = IllegalArgumentException.class) public void shouldNotAcceptNull() { StatementChunker.builder().build().chunk(null); }
private static void addToIndex(CloneIndex index, String resourceId, String sourceCode) { List<Statement> statements = STATEMENT_CHUNKER.chunk(TOKEN_CHUNKER.chunk(sourceCode)); BlockChunker blockChunker = new BlockChunker(2); List<Block> blocks = blockChunker.chunk(resourceId, statements); for (Block block : blocks) { index.insert(block); } }
private void createIndex(Iterable<InputFile> sourceFiles) { TokenChunker tokenChunker = JavaTokenProducer.build(); StatementChunker statementChunker = JavaStatementBuilder.build(); BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE); for (InputFile inputFile : sourceFiles) { LOG.debug("Populating index from {}", inputFile); String resourceEffectiveKey = ((DefaultInputFile) inputFile).key(); List<Statement> statements; try(Reader reader = new InputStreamReader(new FileInputStream(inputFile.file()), fs.encoding())) { statements = statementChunker.chunk(tokenChunker.chunk(reader)); } catch (FileNotFoundException e) { throw new IllegalStateException("Cannot find file " + inputFile.file(), e); } catch (IOException e ) { throw new IllegalStateException("Exception hnadling file: " + inputFile.file(), e); } List<Block> blocks = blockChunker.chunk(resourceEffectiveKey, statements); index.insert(inputFile, blocks); } } }
private SonarDuplicationsIndex createIndex(@Nullable Project project, String language, Iterable<InputFile> sourceFiles) { final SonarDuplicationsIndex index = indexFactory.create(project, language); TokenChunker tokenChunker = JavaTokenProducer.build(); StatementChunker statementChunker = JavaStatementBuilder.build(); BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE); for (InputFile inputFile : sourceFiles) { LOG.debug("Populating index from {}", inputFile); String resourceEffectiveKey = ((DeprecatedDefaultInputFile) inputFile).key(); List<Statement> statements; Reader reader = null; try { reader = new InputStreamReader(new FileInputStream(inputFile.file()), fs.encoding()); statements = statementChunker.chunk(tokenChunker.chunk(reader)); } catch (FileNotFoundException e) { throw new SonarException("Cannot find file " + inputFile.file(), e); } finally { IOUtils.closeQuietly(reader); } List<Block> blocks = blockChunker.chunk(resourceEffectiveKey, statements); index.insert(inputFile, blocks); } return index; }
private void createIndex(Iterable<InputFile> sourceFiles) { TokenChunker tokenChunker = JavaTokenProducer.build(); StatementChunker statementChunker = JavaStatementBuilder.build(); BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE); for (InputFile inputFile : sourceFiles) { LOG.debug("Populating index from {}", inputFile); String resourceEffectiveKey = inputFile.key(); List<Statement> statements; try (InputStream is = inputFile.inputStream(); Reader reader = new InputStreamReader(is, inputFile.charset())) { statements = statementChunker.chunk(tokenChunker.chunk(reader)); } catch (FileNotFoundException e) { throw new IllegalStateException("Cannot find file " + inputFile.file(), e); } catch (IOException e) { throw new IllegalStateException("Exception handling file: " + inputFile.file(), e); } List<Block> blocks; try { blocks = blockChunker.chunk(resourceEffectiveKey, statements); } catch (Exception e) { throw new IllegalStateException("Cannot process file " + inputFile.file(), e); } index.insert(inputFile, blocks); } }