static DtoCache forActivityDtos(DbClient dbClient, DbSession dbSession, Collection<CeActivityDto> ceActivityDtos) { Map<String, ComponentDto> componentsByUuid = dbClient.componentDao().selectByUuids( dbSession, getComponentUuidsOfCeActivities(ceActivityDtos)) .stream() .collect(uniqueIndex(ComponentDto::uuid)); Multimap<String, CeTaskCharacteristicDto> characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao() .selectByTaskUuids(dbSession, ceActivityDtos.stream().map(CeActivityDto::getUuid).collect(Collectors.toList())) .stream().collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid)); Set<String> submitterUuids = ceActivityDtos.stream().map(CeActivityDto::getSubmitterUuid).filter(Objects::nonNull).collect(toSet()); Map<String, UserDto> usersByUuid = dbClient.userDao().selectByUuids(dbSession, submitterUuids).stream().collect(uniqueIndex(UserDto::getUuid)); return new DtoCache(componentsByUuid, buildOrganizationsByUuid(dbClient, dbSession, componentsByUuid), characteristicsByTaskUuid, usersByUuid); }
static DtoCache forQueueDtos(DbClient dbClient, DbSession dbSession, Collection<CeQueueDto> ceQueueDtos) { Map<String, ComponentDto> componentsByUuid = dbClient.componentDao().selectByUuids(dbSession, componentUuidsOfCeQueues(ceQueueDtos)) .stream() .collect(uniqueIndex(ComponentDto::uuid)); Multimap<String, CeTaskCharacteristicDto> characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao() .selectByTaskUuids(dbSession, ceQueueDtos.stream().map(CeQueueDto::getUuid).collect(Collectors.toList())) .stream().collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid)); Set<String> submitterUuids = ceQueueDtos.stream().map(CeQueueDto::getSubmitterUuid).filter(Objects::nonNull).collect(toSet()); Map<String, UserDto> usersByUuid = dbClient.userDao().selectByUuids(dbSession, submitterUuids).stream().collect(uniqueIndex(UserDto::getUuid)); return new DtoCache(componentsByUuid, buildOrganizationsByUuid(dbClient, dbSession, componentsByUuid), characteristicsByTaskUuid, usersByUuid); }
private List<CeTask> loadTasks(DbSession dbSession, List<CeQueueDto> dtos) { // load components, if defined Set<String> componentUuids = dtos.stream() .flatMap(dto -> Stream.of(dto.getComponentUuid(), dto.getMainComponentUuid())) .filter(Objects::nonNull) .collect(Collectors.toSet()); Map<String, ComponentDto> componentsByUuid = dbClient.componentDao() .selectByUuids(dbSession, componentUuids).stream() .collect(uniqueIndex(ComponentDto::uuid)); // load characteristics // TODO could be avoided, characteristics are already present in submissions Set<String> taskUuids = dtos.stream().map(CeQueueDto::getUuid).collect(MoreCollectors.toSet(dtos.size())); Multimap<String, CeTaskCharacteristicDto> characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao() .selectByTaskUuids(dbSession, taskUuids).stream() .collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid)); List<CeTask> result = new ArrayList<>(); for (CeQueueDto dto : dtos) { ComponentDto component = ofNullable(dto.getComponentUuid()) .map(componentsByUuid::get) .orElse(null); ComponentDto mainComponent = ofNullable(dto.getMainComponentUuid()) .map(componentsByUuid::get) .orElse(null); Map<String, String> characteristics = characteristicsByTaskUuid.get(dto.getUuid()).stream() .collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue)); result.add(convertToTask(dbSession, dto, characteristics, component, mainComponent)); } return result; }
@Override public Optional<CeTask> peek(String workerUuid) { requireNonNull(workerUuid, "workerUuid can't be null"); if (computeEngineStatus.getStatus() != ComputeEngineStatus.Status.STARTED || getWorkersPauseStatus() != WorkersPauseStatus.RESUMED) { return Optional.empty(); } try (DbSession dbSession = dbClient.openSession(false)) { CeQueueDao ceQueueDao = dbClient.ceQueueDao(); int i = ceQueueDao.resetToPendingForWorker(dbSession, workerUuid); if (i > 0) { dbSession.commit(); LOG.debug("{} in progress tasks reset for worker uuid {}", i, workerUuid); } Optional<CeQueueDto> opt = ceQueueDao.peek(dbSession, workerUuid); if (opt.isPresent()) { CeQueueDto taskDto = opt.get(); Map<String, ComponentDto> componentsByUuid = loadComponentDtos(dbSession, taskDto); Map<String, String> characteristics = dbClient.ceTaskCharacteristicsDao().selectByTaskUuids(dbSession, singletonList(taskDto.getUuid())).stream() .collect(uniqueIndex(CeTaskCharacteristicDto::getKey, CeTaskCharacteristicDto::getValue)); CeTask task = convertToTask(dbSession, taskDto, characteristics, ofNullable(taskDto.getComponentUuid()).map(componentsByUuid::get).orElse(null), ofNullable(taskDto.getMainComponentUuid()).map(componentsByUuid::get).orElse(null)); queueStatus.addInProgress(); return Optional.of(task); } return Optional.empty(); } }
@Override public void start() { try (DbSession dbSession = dbClient.openSession(false)) { Date sixMonthsAgo = DateUtils.addDays(new Date(system2.now()), -180); LOGGER.info("Delete the Compute Engine tasks created before {}", sixMonthsAgo.getTime()); Set<String> ceActivityUuids = dbClient.ceActivityDao().selectOlderThan(dbSession, sixMonthsAgo.getTime()) .stream() .map(CeActivityDto::getUuid) .collect(toSet()); dbClient.ceActivityDao().deleteByUuids(dbSession, ceActivityUuids); dbClient.ceTaskCharacteristicsDao().deleteByTaskUuids(dbSession, ceActivityUuids); dbClient.ceTaskInputDao().deleteByUuids(dbSession, ceActivityUuids); Date fourWeeksAgo = DateUtils.addDays(new Date(system2.now()), -28); LOGGER.info("Delete the Scanner contexts tasks created before {}", fourWeeksAgo.getTime()); Set<String> scannerContextUuids = dbClient.ceScannerContextDao().selectOlderThan(dbSession, fourWeeksAgo.getTime()); dbClient.ceScannerContextDao().deleteByUuids( dbSession, concat(ceActivityUuids.stream(), scannerContextUuids.stream()).collect(toSet())); dbSession.commit(); } }
private List<CeTaskCharacteristicDto> selectTaskCharecteristic(String taskUuid) { return dbTester.getDbClient().ceTaskCharacteristicsDao().selectByTaskUuids(dbTester.getSession(), Collections.singletonList(taskUuid)); }
private void insertCeTaskCharacteristics(String uuid, int count) { List<CeTaskCharacteristicDto> dtos = IntStream.range(0, count) .mapToObj(i -> new CeTaskCharacteristicDto() .setUuid(UuidFactoryFast.getInstance().create()) .setTaskUuid(uuid) .setKey("key_" + uuid.hashCode() + i) .setValue("value_" + uuid.hashCode() + i)) .collect(Collectors.toList()); dbClient.ceTaskCharacteristicsDao().insert(dbSession, dtos); dbSession.commit(); }
private CeQueueDto addToQueueInDb(DbSession dbSession, CeTaskSubmit submission) { for (Map.Entry<String, String> characteristic : submission.getCharacteristics().entrySet()) { CeTaskCharacteristicDto characteristicDto = new CeTaskCharacteristicDto(); characteristicDto.setUuid(uuidFactory.create()); characteristicDto.setTaskUuid(submission.getUuid()); characteristicDto.setKey(characteristic.getKey()); characteristicDto.setValue(characteristic.getValue()); dbClient.ceTaskCharacteristicsDao().insert(dbSession, characteristicDto); } CeQueueDto dto = new CeQueueDto(); dto.setUuid(submission.getUuid()); dto.setTaskType(submission.getType()); submission.getComponent().ifPresent(component -> dto .setComponentUuid(component.getUuid()) .setMainComponentUuid(component.getMainComponentUuid())); dto.setStatus(PENDING); dto.setSubmitterUuid(submission.getSubmitterUuid()); dbClient.ceQueueDao().insert(dbSession, dto); return dto; }
private CeTaskCharacteristicDto insertCharacteristic(String taskUuid, String key, String value) { CeTaskCharacteristicDto dto = new CeTaskCharacteristicDto() .setUuid(Uuids.createFast()) .setTaskUuid(taskUuid) .setKey(key) .setValue(value); db.getDbClient().ceTaskCharacteristicsDao().insert(db.getSession(), Collections.singletonList(dto)); db.commit(); return dto; }
private CeTaskCharacteristicDto insertCharacteristic(String taskUuid, String key, String value) { CeTaskCharacteristicDto dto = new CeTaskCharacteristicDto() .setUuid(Uuids.createFast()) .setTaskUuid(taskUuid) .setKey(key) .setValue(value); db.getDbClient().ceTaskCharacteristicsDao().insert(db.getSession(), Collections.singletonList(dto)); db.commit(); return dto; } }
private CeTaskCharacteristicDto insertCharacteristic(String taskUuid, String key, String value) { CeTaskCharacteristicDto dto = new CeTaskCharacteristicDto() .setUuid(Uuids.createFast()) .setTaskUuid(taskUuid) .setKey(key) .setValue(value); db.getDbClient().ceTaskCharacteristicsDao().insert(db.getSession(), Collections.singletonList(dto)); db.commit(); return dto; }
private void insertWithDate(String uuid, LocalDateTime dateTime) { long date = dateTime.toInstant(UTC).toEpochMilli(); CeQueueDto queueDto = new CeQueueDto(); queueDto.setUuid(uuid); queueDto.setTaskType(CeTaskTypes.REPORT); CeActivityDto dto = new CeActivityDto(queueDto); dto.setStatus(CeActivityDto.Status.SUCCESS); when(system2.now()).thenReturn(date); CeTaskCharacteristicDto ceTaskCharacteristicDto = new CeTaskCharacteristicDto() .setUuid(UuidFactoryFast.getInstance().create()) .setValue(randomAlphanumeric(10)) .setKey(randomAlphanumeric(10)) .setTaskUuid(dto.getUuid()); dbTester.getDbClient().ceTaskInputDao().insert(dbTester.getSession(), dto.getUuid(), IOUtils.toInputStream(randomAlphanumeric(10), Charset.forName("UTF-8"))); dbTester.getDbClient().ceActivityDao().insert(dbTester.getSession(), dto); dbTester.getDbClient().ceTaskCharacteristicsDao().insert(dbTester.getSession(), Collections.singletonList(ceTaskCharacteristicDto)); dbTester.getSession().commit(); insertScannerContext(uuid, date); }
static DtoCache forQueueDtos(DbClient dbClient, DbSession dbSession, Collection<CeQueueDto> ceQueueDtos) { Map<String, ComponentDto> componentsByUuid = dbClient.componentDao().selectByUuids(dbSession, componentUuidsOfCeQueues(ceQueueDtos)) .stream() .collect(uniqueIndex(ComponentDto::uuid)); Multimap<String, CeTaskCharacteristicDto> characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao() .selectByTaskUuids(dbSession, ceQueueDtos.stream().map(CeQueueDto::getUuid).collect(Collectors.toList())) .stream().collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid)); Set<String> submitterUuids = ceQueueDtos.stream().map(CeQueueDto::getSubmitterUuid).filter(Objects::nonNull).collect(toSet()); Map<String, UserDto> usersByUuid = dbClient.userDao().selectByUuids(dbSession, submitterUuids).stream().collect(uniqueIndex(UserDto::getUuid)); return new DtoCache(componentsByUuid, buildOrganizationsByUuid(dbClient, dbSession, componentsByUuid), characteristicsByTaskUuid, usersByUuid); }
static DtoCache forActivityDtos(DbClient dbClient, DbSession dbSession, Collection<CeActivityDto> ceActivityDtos) { Map<String, ComponentDto> componentsByUuid = dbClient.componentDao().selectByUuids( dbSession, getComponentUuidsOfCeActivities(ceActivityDtos)) .stream() .collect(uniqueIndex(ComponentDto::uuid)); Multimap<String, CeTaskCharacteristicDto> characteristicsByTaskUuid = dbClient.ceTaskCharacteristicsDao() .selectByTaskUuids(dbSession, ceActivityDtos.stream().map(CeActivityDto::getUuid).collect(Collectors.toList())) .stream().collect(MoreCollectors.index(CeTaskCharacteristicDto::getTaskUuid)); Set<String> submitterUuids = ceActivityDtos.stream().map(CeActivityDto::getSubmitterUuid).filter(Objects::nonNull).collect(toSet()); Map<String, UserDto> usersByUuid = dbClient.userDao().selectByUuids(dbSession, submitterUuids).stream().collect(uniqueIndex(UserDto::getUuid)); return new DtoCache(componentsByUuid, buildOrganizationsByUuid(dbClient, dbSession, componentsByUuid), characteristicsByTaskUuid, usersByUuid); }
private CeTask submitReport(DbSession dbSession, InputStream reportInput, ComponentDto project, Map<String, String> characteristicsMap) { CeTaskSubmit.Builder submit = queue.prepareSubmit(); List<CeTaskCharacteristicDto> characteristics = characteristicsMap.entrySet().stream() .map(e -> toDto(submit.getUuid(), e.getKey(), e.getValue())).collect(Collectors.toList()); // the report file must be saved before submitting the task dbClient.ceTaskInputDao().insert(dbSession, submit.getUuid(), reportInput); if (!characteristics.isEmpty()) { dbClient.ceTaskCharacteristicsDao().insert(dbSession, characteristics); } dbSession.commit(); submit.setType(CeTaskTypes.REPORT); submit.setComponentUuid(project.uuid()); submit.setSubmitterUuid(userSession.getUuid()); return queue.submit(submit.build()); }