private void cleanUpTaskInputOrphans() { try (DbSession dbSession = dbClient.openSession(false)) { // Reports that have been processed are not kept in database yet. // They are supposed to be systematically dropped. // Let's clean-up orphans if any. List<String> uuids = dbClient.ceTaskInputDao().selectUuidsNotInQueue(dbSession); dbClient.ceTaskInputDao().deleteByUuids(dbSession, uuids); dbSession.commit(); } }
private void insertCeTaskInput(String uuid) { dbClient.ceTaskInputDao().insert(dbSession, uuid, new ByteArrayInputStream("some content man!".getBytes())); dbSession.commit(); }
@Override public void start() { try (DbSession dbSession = dbClient.openSession(false)) { Date sixMonthsAgo = DateUtils.addDays(new Date(system2.now()), -180); LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime()); Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTime()) .stream() .map(CeActivityDto::getUuid) .collect(toSet()); dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids); dbClient.ceTaskCharacteristicsDao().deleteByTaskUuids(dbSession, ceActivityUuids); dbClient.ceTaskInputDao().deleteByUuids(dbSession, ceActivityUuids); Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28); LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime()); Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTime()); dbClient.ceScannerContextDao().deleteByUuids( dbSession, concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet())); dbSession.commit(); } }
@Override public void execute(ComputationStep.Context context) { try (DbSession dbSession = dbClient.openSession(false)) { Optional<CeTaskInputDao.DataStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid()); if (opt.isPresent()) { File unzippedDir = tempFolder.newDir(); try (CeTaskInputDao.DataStream reportStream = opt.get(); InputStream zipStream = new BufferedInputStream(reportStream.getInputStream())) { ZipUtils.unzip(zipStream, unzippedDir); } catch (IOException e) { throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); } reportDirectoryHolder.setDirectory(unzippedDir); if (LOGGER.isDebugEnabled()) { // size is not added to context statistics because computation // can take time. It's enabled only if log level is DEBUG. try { String dirSize = FileUtils.byteCountToDisplaySize(FileUtils2.sizeOf(unzippedDir.toPath())); LOGGER.debug("Analysis report is {} uncompressed", dirSize); } catch (IOException e) { LOGGER.warn("Fail to compute size of directory " + unzippedDir, e); } } } else { throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database"); } } }
private Optional<CeTaskInputDao.DataStream> selectTaskInput(String taskUuid) { return dbTester.getDbClient().ceTaskInputDao().selectData(dbTester.getSession(), taskUuid); }
protected void remove(DbSession dbSession, CeQueueDto queueDto, CeActivityDto activityDto) { String taskUuid = queueDto.getUuid(); CeQueueDto.Status expectedQueueDtoStatus = queueDto.getStatus(); dbClient.ceActivityDao().insert(dbSession, activityDto); dbClient.ceTaskInputDao().deleteByUuids(dbSession, singleton(taskUuid)); int deletedTasks = dbClient.ceQueueDao().deleteByUuid(dbSession, taskUuid, new DeleteIf(expectedQueueDtoStatus)); if (deletedTasks == 1) { dbSession.commit(); } else { Loggers.get(CeQueueImpl.class).debug( "Remove rolled back because task in queue with uuid {} and status {} could not be deleted", taskUuid, expectedQueueDtoStatus); dbSession.rollback(); } }
private void insertTaskData(String taskUuid) { dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), taskUuid, IOUtils.toInputStream("{binary}")); dbTester.getSession().commit(); }
private CeTask submitReport(DbSession dbSession, InputStream reportInput, ComponentDto project, Map<String, String> characteristics) { CeTaskSubmit.Builder submit = queue.prepareSubmit(); // the report file must be saved before submitting the task dbClient.ceTaskInputDao().insert(dbSession, submit.getUuid(), reportInput); dbSession.commit(); submit.setType(CeTaskTypes.REPORT); submit.setComponent(CeTaskSubmit.Component.fromDto(project)); submit.setSubmitterUuid(userSession.getUuid()); submit.setCharacteristics(characteristics); return queue.submit(submit.build()); }
@Test public void unzip_report() throws Exception { logTester.setLevel(LoggerLevel.DEBUG); File reportFile = generateReport(); try (InputStream input = FileUtils.openInputStream(reportFile)) { dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), TASK_UUID, input); } dbTester.getSession().commit(); dbTester.getSession().close(); underTest.execute(new TestComputationStepContext()); // directory contains the uncompressed report (which contains only metadata.pb in this test) File unzippedDir = reportDirectoryHolder.getDirectory(); assertThat(unzippedDir).isDirectory().exists(); assertThat(unzippedDir.listFiles()).hasSize(1); assertThat(new File(unzippedDir, "metadata.pb")).hasContent("{metadata}"); assertThat(logTester.logs(LoggerLevel.DEBUG)).anyMatch(log -> log.matches("Analysis report is \\d+ bytes uncompressed")); }
@Test public void start_deletes_orphan_report_files() { // analysis reports are persisted but the associated // task is not in the queue insertInQueue("TASK_1", CeQueueDto.Status.PENDING); insertTaskData("TASK_1"); insertTaskData("TASK_2"); runCleaner(); CeTaskInputDao dataDao = dbTester.getDbClient().ceTaskInputDao(); Optional<CeTaskInputDao.DataStream> task1Data = dataDao.selectData(dbTester.getSession(), "TASK_1"); assertThat(task1Data).isPresent(); task1Data.get().close(); assertThat(dataDao.selectData(dbTester.getSession(), "TASK_2")).isNotPresent(); }
private void insertWithDate(String uuid, LocalDateTime dateTime) { long date = dateTime.toInstant(UTC).toEpochMilli(); CeQueueDto queueDto = new CeQueueDto(); queueDto.setUuid(uuid); queueDto.setTaskType(CeTaskTypes.REPORT); CeActivityDto dto = new CeActivityDto(queueDto); dto.setStatus(CeActivityDto.Status.SUCCESS); when(system2.now()).thenReturn(date); CeTaskCharacteristicDto ceTaskCharacteristicDto = new CeTaskCharacteristicDto() .setUuid(UuidFactoryFast.getInstance().create()) .setValue(randomAlphanumeric(10)) .setKey(randomAlphanumeric(10)) .setTaskUuid(dto.getUuid()); dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), dto.getUuid(), IOUtils.toInputStream(randomAlphanumeric(10), Charset.forName("UTF-8"))); dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto); dbTester.getDbClient().ceTaskCharacteristicsDao().insert(dbTester.getSession(), Collections.singletonList(ceTaskCharacteristicDto)); dbTester.getSession().commit(); insertScannerContext(uuid, date); }
private void verifyConsistency(DbSession dbSession) { // Reports that have been processed are not kept in database yet. // They are supposed to be systematically dropped. // Let's clean-up orphans if any. List<String> uuids = dbClient.ceTaskInputDao().selectUuidsNotInQueue(dbSession); dbClient.ceTaskInputDao().deleteByUuids(dbSession, uuids); dbSession.commit(); }
private CeTask submitReport(DbSession dbSession, InputStream reportInput, ComponentDto project, Map<String, String> characteristicsMap) { CeTaskSubmit.Builder submit = queue.prepareSubmit(); List<CeTaskCharacteristicDto> characteristics = characteristicsMap.entrySet().stream() .map(e -> toDto(submit.getUuid(), e.getKey(), e.getValue())).collect(Collectors.toList()); // the report file must be saved before submitting the task dbClient.ceTaskInputDao().insert(dbSession, submit.getUuid(), reportInput); if (!characteristics.isEmpty()) { dbClient.ceTaskCharacteristicsDao().insert(dbSession, characteristics); } dbSession.commit(); submit.setType(CeTaskTypes.REPORT); submit.setComponentUuid(project.uuid()); submit.setSubmitterUuid(userSession.getUuid()); return queue.submit(submit.build()); }
@Override public void execute() { try (DbSession dbSession = dbClient.openSession(false)) { Optional<CeTaskInputDao.DataStream> opt = dbClient.ceTaskInputDao().selectData(dbSession, task.getUuid()); if (opt.isPresent()) { File unzippedDir = tempFolder.newDir(); try (CeTaskInputDao.DataStream reportStream = opt.get(); InputStream zipStream = new BufferedInputStream(reportStream.getInputStream())) { ZipUtils.unzip(zipStream, unzippedDir); } catch (IOException e) { throw new IllegalStateException("Fail to extract report " + task.getUuid() + " from database", e); } reportDirectoryHolder.setDirectory(unzippedDir); } else { throw MessageException.of("Analysis report " + task.getUuid() + " is missing in database"); } } }
protected void remove(DbSession dbSession, CeQueueDto queueDto, CeActivityDto activityDto) { dbClient.ceActivityDao().insert(dbSession, activityDto); dbClient.ceQueueDao().deleteByUuid(dbSession, queueDto.getUuid()); dbClient.ceTaskInputDao().deleteByUuids(dbSession, singleton(queueDto.getUuid())); dbSession.commit(); }