@Test public void persist_scm_some_lines() { DbFileSources.Data dbData = DbFileSources.Data.newBuilder().addAllLines(Arrays.asList( DbFileSources.Line.newBuilder() .setScmAuthor("john") assertThat(data.getLinesList()).hasSize(3); assertThat(data.getLines(0).getScmAuthor()).isEqualTo("john"); assertThat(data.getLines(0).getScmDate()).isEqualTo(123456789L); assertThat(data.getLines(0).getScmRevision()).isEqualTo("rev-1"); assertThat(data.getLines(1).getScmAuthor()).isEmpty(); assertThat(data.getLines(1).getScmDate()).isEqualTo(223456789L); assertThat(data.getLines(1).getScmRevision()).isEmpty(); assertThat(data.getLines(2).getScmAuthor()).isEmpty(); assertThat(data.getLines(2).getScmDate()).isEqualTo(0); assertThat(data.getLines(2).getScmRevision()).isEmpty(); verify(fileSourceDataWarnings).commitWarnings();
@Test public void persist_sources() { List<String> lineHashes = Arrays.asList("137f72c3708c6bd0de00a0e5a69c699b", "e6251bcf1a7dc3ba5e7933e325bbe605"); String sourceHash = "ee5a58024a155466b43bc559d953e018"; DbFileSources.Data fileSourceData = DbFileSources.Data.newBuilder() .addAllLines(Arrays.asList( DbFileSources.Line.newBuilder().setSource("line1").setLine(1).build(), DbFileSources.Line.newBuilder().setSource("line2").setLine(2).build())) .build(); when(fileSourceDataComputer.compute(fileComponent().build(), fileSourceDataWarnings)) .thenReturn(new FileSourceDataComputer.Data(fileSourceData, lineHashes, sourceHash, null)); underTest.execute(new TestComputationStepContext()); assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); FileSourceDto fileSourceDto = dbClient.fileSourceDao().selectByFileUuid(session, FILE1_UUID); assertThat(fileSourceDto.getProjectUuid()).isEqualTo(PROJECT_UUID); assertThat(fileSourceDto.getFileUuid()).isEqualTo(FILE1_UUID); assertThat(fileSourceDto.getBinaryData()).isNotEmpty(); assertThat(fileSourceDto.getDataHash()).isNotEmpty(); assertThat(fileSourceDto.getLineHashesVersion()).isEqualTo(LineHashVersion.WITHOUT_SIGNIFICANT_CODE.getDbValue()); assertThat(fileSourceDto.getLineHashes()).isNotEmpty(); assertThat(fileSourceDto.getCreatedAt()).isEqualTo(NOW); assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(NOW); DbFileSources.Data data = fileSourceDto.getSourceData(); assertThat(data.getLinesCount()).isEqualTo(2); assertThat(data.getLines(0).getLine()).isEqualTo(1); assertThat(data.getLines(0).getSource()).isEqualTo("line1"); assertThat(data.getLines(1).getLine()).isEqualTo(2); assertThat(data.getLines(1).getSource()).isEqualTo("line2"); verify(fileSourceDataWarnings).commitWarnings(); }
private static DbFileSources.Data decodeRegularSourceData(byte[] binaryData) throws IOException { try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { return DbFileSources.Data.parseFrom(lz4Input); } catch (InvalidProtocolBufferException e) { if (SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE.equals(e.getMessage())) { return decodeHugeSourceData(binaryData); } throw e; } }
/** * Serialize and compress protobuf message {@link org.sonar.db.protobuf.DbFileSources.Data} * in the column BINARY_DATA. */ public static byte[] encodeSourceData(DbFileSources.Data data) { ByteArrayOutputStream byteOutput = new ByteArrayOutputStream(); LZ4BlockOutputStream compressedOutput = new LZ4BlockOutputStream(byteOutput); try { data.writeTo(compressedOutput); compressedOutput.close(); return byteOutput.toByteArray(); } catch (IOException e) { throw new IllegalStateException("Fail to serialize and compress source data", e); } finally { IOUtils.closeQuietly(compressedOutput); } }
public org.sonar.db.protobuf.DbFileSources.Data buildPartial() { org.sonar.db.protobuf.DbFileSources.Data result = new org.sonar.db.protobuf.DbFileSources.Data(this); int from_bitField0_ = bitField0_; if (linesBuilder_ == null) { if (((bitField0_ & 0x00000001) == 0x00000001)) { lines_ = java.util.Collections.unmodifiableList(lines_); bitField0_ = (bitField0_ & ~0x00000001); } result.lines_ = lines_; } else { result.lines_ = linesBuilder_.build(); } onBuilt(); return result; }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder() {
public Optional<DbScmInfo> getScmInfo(Component file) { Optional<String> uuid = getFileUUid(file); if (!uuid.isPresent()) { return Optional.empty(); } LOGGER.trace("Reading SCM info from DB for file '{}'", uuid.get()); try (DbSession dbSession = dbClient.openSession(false)) { FileSourceDto dto = dbClient.fileSourceDao().selectSourceByFileUuid(dbSession, uuid.get()); if (dto == null) { return Optional.empty(); } return DbScmInfo.create(dto.getSourceData().getLinesList(), dto.getSrcHash()); } }
private <E> Optional<Iterable<E>> getLines(DbSession dbSession, String fileUuid, int from, int toInclusive, Function<DbFileSources.Line, E> function) { verifyLine(from); checkArgument(toInclusive >= from, String.format("Line number must greater than or equal to %d, got %d", from, toInclusive)); FileSourceDto dto = dbClient.fileSourceDao().selectSourceByFileUuid(dbSession, fileUuid); if (dto == null) { return Optional.empty(); } return Optional.of(dto.getSourceData().getLinesList().stream() .filter(line -> line.hasLine() && line.getLine() >= from) .limit((toInclusive - from) + 1L) .map(function) .collect(MoreCollectors.toList())); }
private static DbFileSources.Data decodeHugeSourceData(byte[] binaryData) throws IOException { try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { CodedInputStream input = CodedInputStream.newInstance(lz4Input); input.setSizeLimit(Integer.MAX_VALUE); return DbFileSources.Data.parseFrom(input); } }
private static DbFileSources.Data decodeRegularSourceData(byte[] binaryData) throws IOException { try (LZ4BlockInputStream lz4Input = new LZ4BlockInputStream(new ByteArrayInputStream(binaryData))) { return DbFileSources.Data.parseFrom(lz4Input); } catch (InvalidProtocolBufferException e) { if (SIZE_LIMIT_EXCEEDED_EXCEPTION_MESSAGE.equals(e.getMessage())) { return decodeHugeSourceData(binaryData); } throw e; } }
public Data compute() { DbFileSources.Data.Builder fileSourceBuilder = DbFileSources.Data.newBuilder(); int currentLine = 0; while (linesIterator.hasNext()) { currentLine++; read(fileSourceBuilder, currentLine, linesIterator.next(), linesIterator.hasNext()); } return new Data(fileSourceBuilder.build(), lineHashesComputer.getResult(), sourceHashComputer.getHash()); }
.setLineHashes(Collections.singletonList("6438c669e0d0de98e6929c2cc0fac474")) .setDataHash("6cad150e3d065976c230cddc5a09efaa") .setSourceData(DbFileSources.Data.newBuilder() .addLines(DbFileSources.Line.newBuilder() .setLine(1) dbTester.getSession().commit(); DbFileSources.Data newSourceData = DbFileSources.Data.newBuilder() .addLines(DbFileSources.Line.newBuilder() .setLine(1)
public Data compute(Component file, FileSourceDataWarnings fileSourceDataWarnings) { try (CloseableIterator<String> linesIterator = sourceLinesRepository.readLines(file); SourceLineReadersFactory.LineReaders lineReaders = sourceLineReadersFactory.getLineReaders(file)) { SourceLinesHashRepositoryImpl.LineHashesComputer lineHashesComputer = sourceLinesHash.getLineHashesComputerToPersist(file); DbFileSources.Data.Builder fileSourceBuilder = DbFileSources.Data.newBuilder(); int currentLine = 0; while (linesIterator.hasNext()) { currentLine++; String lineSource = linesIterator.next(); boolean hasNextLine = linesIterator.hasNext(); sourceHashComputer.addLine(lineSource, hasNextLine); lineHashesComputer.addLine(lineSource); DbFileSources.Line.Builder lineBuilder = fileSourceBuilder .addLinesBuilder() .setSource(lineSource) .setLine(currentLine); lineReaders.read(lineBuilder, readError -> fileSourceDataWarnings.addWarning(file, readError)); } Changeset latestChangeWithRevision = lineReaders.getLatestChangeWithRevision(); return new Data(fileSourceBuilder.build(), lineHashesComputer.getResult(), sourceHashComputer.getHash(), latestChangeWithRevision); } }
private void addFileSourceInDb(@Nullable String author, @Nullable Long date, @Nullable String revision, String srcHash, String fileUuid) { DbFileSources.Data.Builder fileDataBuilder = DbFileSources.Data.newBuilder(); DbFileSources.Line.Builder builder = fileDataBuilder.addLinesBuilder() .setLine(1); if (author != null) { builder.setScmAuthor(author); } if (date != null) { builder.setScmDate(date); } if (revision != null) { builder.setScmRevision(revision); } dbTester.getDbClient().fileSourceDao().insert(dbTester.getSession(), new FileSourceDto() .setFileUuid(fileUuid) .setProjectUuid("PROJECT_UUID") .setSourceData(fileDataBuilder.build()) .setSrcHash(srcHash)); dbTester.commit(); }
@Test public void update_sources_when_revision_is_missing() { DbFileSources.Data sourceData = DbFileSources.Data.newBuilder() .addLines(DbFileSources.Line.newBuilder() .setLine(1) .setSource("line") .build()) .build(); dbClient.fileSourceDao().insert(dbTester.getSession(), createDto(dto -> dto.setRevision(null))); dbTester.getSession().commit(); Changeset changeset = Changeset.newChangesetBuilder().setDate(1L).setRevision("revision").build(); setComputedData(sourceData, Collections.singletonList("137f72c3708c6bd0de00a0e5a69c699b"), "29f25900140c94db38035128cb6de6a2", changeset); underTest.execute(new TestComputationStepContext()); assertThat(dbTester.countRowsOfTable("file_sources")).isEqualTo(1); FileSourceDto fileSourceDto = dbClient.fileSourceDao().selectByFileUuid(session, FILE1_UUID); assertThat(fileSourceDto.getCreatedAt()).isEqualTo(PAST); assertThat(fileSourceDto.getUpdatedAt()).isEqualTo(NOW); assertThat(fileSourceDto.getRevision()).isEqualTo("revision"); verify(fileSourceDataWarnings).commitWarnings(); }
private static DbFileSources.Data.Builder newRandomData(int numberOfLines) { DbFileSources.Data.Builder dataBuilder = DbFileSources.Data.newBuilder(); for (int i = 1; i <= numberOfLines; i++) { dataBuilder.addLinesBuilder() .setLine(i) .setScmRevision(randomAlphanumeric(15)) .setScmAuthor(randomAlphanumeric(10)) .setScmDate(RandomUtils.nextLong()) .setSource(randomAlphanumeric(20)) .setLineHits(RandomUtils.nextInt(4)) .setConditions(RandomUtils.nextInt(4)) .setCoveredConditions(RandomUtils.nextInt(4)) .setHighlighting(randomAlphanumeric(40)) .setSymbols(randomAlphanumeric(30)) .addAllDuplication(Arrays.asList(RandomUtils.nextInt(200), RandomUtils.nextInt(200))) .build(); } return dataBuilder; } }
@Test public void use_deprecated_overall_coverage_fields_if_exists() throws Exception { DbFileSources.Data.Builder dataBuilder = DbFileSources.Data.newBuilder(); ComponentDto file = insertFileWithData(dataBuilder.addLines(newLineBuilder() .setDeprecatedOverallLineHits(1) .setDeprecatedOverallConditions(2) .setDeprecatedOverallCoveredConditions(3) .setDeprecatedUtLineHits(1) .setDeprecatedUtConditions(2) .setDeprecatedUtCoveredConditions(3) .setDeprecatedItLineHits(1) .setDeprecatedItConditions(2) .setDeprecatedItCoveredConditions(3)).build(), privateProject); setUserWithValidPermission(file); wsTester .newGetRequest("api/sources", "lines") .setParam("uuid", file.uuid()) .execute() .assertJson(getClass(), "convert_deprecated_data.json"); }
@Test public void nothing_to_do_when_no_coverage_info_for_next_line() { CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList( ScannerReport.LineCoverage.newBuilder() .setLine(1) .setConditions(10) .setHits(true) .setCoveredConditions(2) .build() // No coverage info on line 2 ).iterator()); DbFileSources.Data.Builder fileSourceBuilder = DbFileSources.Data.newBuilder(); DbFileSources.Line.Builder line1Builder = fileSourceBuilder.addLinesBuilder().setLine(1); DbFileSources.Line.Builder line2Builder = fileSourceBuilder.addLinesBuilder().setLine(2); computeCoverageLine.read(line1Builder); computeCoverageLine.read(line2Builder); assertThat(line2Builder.hasLineHits()).isFalse(); assertThat(line2Builder.hasConditions()).isFalse(); assertThat(line2Builder.hasCoveredConditions()).isFalse(); }
@Test public void does_not_set_deprecated_coverage_fields() { CoverageLineReader computeCoverageLine = new CoverageLineReader(newArrayList(ScannerReport.LineCoverage.newBuilder() .setLine(1) .setConditions(10) .setHits(true) .setCoveredConditions(2) .build()).iterator()); DbFileSources.Line.Builder lineBuilder = DbFileSources.Data.newBuilder().addLinesBuilder().setLine(1); assertThat(computeCoverageLine.read(lineBuilder)).isEmpty(); assertThat(lineBuilder.hasDeprecatedUtLineHits()).isFalse(); assertThat(lineBuilder.hasDeprecatedUtConditions()).isFalse(); assertThat(lineBuilder.hasDeprecatedUtCoveredConditions()).isFalse(); assertThat(lineBuilder.hasDeprecatedOverallLineHits()).isFalse(); assertThat(lineBuilder.hasDeprecatedOverallConditions()).isFalse(); assertThat(lineBuilder.hasDeprecatedOverallCoveredConditions()).isFalse(); assertThat(lineBuilder.hasDeprecatedItLineHits()).isFalse(); assertThat(lineBuilder.hasDeprecatedItConditions()).isFalse(); assertThat(lineBuilder.hasDeprecatedItCoveredConditions()).isFalse(); }
@Test public void group_lines_by_commit() throws Exception { userSessionRule.addProjectPermission(UserRole.CODEVIEWER, project, file); // lines 1 and 2 are the same commit, but not 3 (different date) dbTester.getDbClient().fileSourceDao().insert(dbSession, new FileSourceDto() .setProjectUuid(PROJECT_UUID) .setFileUuid(FILE_UUID) .setSourceData(DbFileSources.Data.newBuilder() .addLines(newSourceLine("julien", "123-456-789", DateUtils.parseDateTime("2015-03-30T12:34:56+0000"), 1)) .addLines(newSourceLine("julien", "123-456-789", DateUtils.parseDateTime("2015-03-30T12:34:56+0000"), 2)) .addLines(newSourceLine("julien", "456-789-101", DateUtils.parseDateTime("2015-03-27T12:34:56+0000"), 3)) .addLines(newSourceLine("simon", "789-101-112", DateUtils.parseDateTime("2015-03-31T12:34:56+0000"), 4)) .build())); dbSession.commit(); WsTester.TestRequest request = tester.newGetRequest("api/sources", "scm").setParam("key", FILE_KEY).setParam("commits_by_line", "false"); request.execute().assertJson(getClass(), "group_lines_by_commit.json"); }