@Override public boolean completeEquals(final Object obj) { return Task.class.isInstance(obj) && super.completeEquals(obj) && DMPPersistenceUtil.getDataModelUtils().completeEquals(((Task) obj).getInputDataModel(), getInputDataModel()) && DMPPersistenceUtil.getDataModelUtils().completeEquals(((Task) obj).getOutputDataModel(), getOutputDataModel()) && DMPPersistenceUtil.getJobUtils().completeEquals(((Task) obj).getJob(), getJob()); } }
@Override public FilterMorphScriptBuilder apply(final Task task) throws DMPConverterException { final Optional<Filter> optionalSkipFilter = Optional.ofNullable(task.getJob().getSkipFilter()); if (!optionalSkipFilter.isPresent()) { LOG.debug("there is no skip filter at job '{}' of task '{}'", task.getJob().getUuid(), task.getUuid()); return this; } final Filter skipFilter = optionalSkipFilter.get(); final Optional<String> optionalSkipFilterExpression = Optional.ofNullable(skipFilter.getExpression()); if (!optionalSkipFilterExpression.isPresent()) { LOG.debug("there is no filter expression in skip filter '{}' of job '{}' of task '{}'", skipFilter.getUuid(), task.getJob().getUuid(), task.getUuid()); return this; } final String filterExpressionString = optionalSkipFilterExpression.get(); final Map<String, FilterExpression> filterExpressionMap = extractFilterExpressions(filterExpressionString); if (filterExpressionMap == null || filterExpressionMap.isEmpty()) { LOG.debug("there are no filter conditions in filter expression in skip filter '{}' of job '{}' of task '{}'", skipFilter.getUuid(), task.getJob().getUuid(), task.getUuid()); return this; } super.apply(task); metaName.setTextContent(SKIP_FILTER + task.getJob().getUuid()); final String attributePathStringXMLEscaped = Iterators.getLast(filterExpressionMap.keySet().iterator()); addFilter(attributePathStringXMLEscaped, COMBINE_VALUE_VARIABLE, filterExpressionMap, rules, false); return this; }
@Test public void simpleTaskTest() { final Job job = createJob(); final DataModel inputDataModel = createDataModel(); final DataModel outputDataModel = createDataModel(); final String taskUUID = UUIDService.getUUID(Task.class.getSimpleName()); final Task task = new Task(taskUUID); task.setName("my task"); task.setDescription("my task description"); task.setJob(job); task.setInputDataModel(inputDataModel); task.setOutputDataModel(outputDataModel); String json = null; try { json = objectMapper.writeValueAsString(task); } catch (final JsonProcessingException e) { e.printStackTrace(); } TaskTest.LOG.debug("task json: " + json); }
@Override public MorphScriptBuilder apply(final Task task) throws DMPConverterException { super.apply(task); final List<String> metas = new ArrayList<>(); final Optional<Map<String, Integer>> optionalInputSchemaMap = generateInputSchemaMap(task); final Optional<DataModel> optionalInputDataModel = Optional.ofNullable(task) .flatMap(task2 -> Optional.ofNullable(task2.getInputDataModel())); final Optional<Schema> optionalInputSchema = optionalInputDataModel .flatMap(inputDataModel -> Optional.ofNullable(inputDataModel.getSchema())); final boolean isXmlSchema = isXmlSchema(optionalInputDataModel); final List<Tuple5<Optional<String>, javaslang.collection.List<String>, String, Element, Optional<String>>> mappingOutputs = new ArrayList<>(); for (final Mapping mapping : task.getJob().getMappings()) { metas.add(MorphScriptBuilder.MAPPING_PREFIX + mapping.getUuid()); final Optional<String> optionalDeepestMappingInput = determineDeepestMappingInputAttributePath(mapping, optionalInputSchemaMap); final Optional<Boolean> optionalSelectValueFromSameSubEntity = determineFilterUseCase(mapping, optionalInputSchema); final Optional<String> optionalCommonAttributePathOfMappingInputs = determineCommonAttributePathOfMappingInputs(mapping, optionalInputSchema, isXmlSchema); createTransformation(rules, mapping, optionalDeepestMappingInput, optionalSelectValueFromSameSubEntity, optionalCommonAttributePathOfMappingInputs) .ifPresent(currentMappingOutputs -> currentMappingOutputs .forEach(currentMappingOutput -> mappingOutputs.add(Tuple.of(currentMappingOutput._1, currentMappingOutput._2, currentMappingOutput._3, currentMappingOutput._4, optionalCommonAttributePathOfMappingInputs)))); createLookupTable(maps, mapping.getTransformation()); } createMappingOutputs(mappingOutputs, rules, doc); metaName.setTextContent(Joiner.on(", ").join(metas)); return this; }
@Test public void testSqlDbRequestTask() throws Exception { final Task task = getTask("sqldbrequest.task.json"); final Job job = task.getJob(); final Set<Mapping> mappings = job.getMappings(); final Mapping mapping = mappings.iterator().next(); final Component mappingTransformationComponent = mapping.getTransformation(); final Transformation mappingTransformationComponentFunction = (Transformation) mappingTransformationComponent.getFunction(); final Set<Component> mappingTransformationComponentFunctionComponents = mappingTransformationComponentFunction.getComponents(); final Component sqlMapLookup = mappingTransformationComponentFunctionComponents.iterator().next(); final Map<String, String> sqlMapLookupParameterMappings = sqlMapLookup.getParameterMappings(); final String user = readManuallyFromTypeSafeConfig("dswarm.db.metadata.username"); final String pass = readManuallyFromTypeSafeConfig("dswarm.db.metadata.password"); final String db = readManuallyFromTypeSafeConfig("dswarm.db.metadata.schema"); setSqlMapParameterMappings(sqlMapLookupParameterMappings, user, pass, db); testXMLTaskWithTuples("sqldbrequest.task.result.json", "sqldbrequest.mabxml.tuples.json", task); }
private static Optional<Map<String, Integer>> generateInputSchemaMap(final Task task) { return Optional.ofNullable(task) .flatMap(task1 -> Optional.ofNullable(task1.getInputDataModel())) .flatMap(inputDataModel -> Optional.ofNullable(inputDataModel.getSchema())) .flatMap(MorphScriptBuilder::generateSchemaMap); }
default TRANSFORMATIONFLOW fromTask(final Task task) throws DMPConverterException { final String morphScriptString = new MorphScriptBuilder().apply(task).toString(); final Optional<String> maybeFilterScript = Optional.ofNullable( new FilterMorphScriptBuilder().apply(task).toString()); return fromAnything( readString(morphScriptString), readString(maybeFilterScript), Optional.ofNullable(task.getOutputDataModel())); }
@Test public void testSqlMapTask() throws Exception { final Task task = getTask("sqlmap.lookup.task.json"); final Job job = task.getJob(); final Set<Mapping> mappings = job.getMappings(); final Mapping mapping = mappings.iterator().next(); final Component mappingTransformationComponent = mapping.getTransformation(); final Transformation mappingTransformationComponentFunction = (Transformation) mappingTransformationComponent.getFunction(); final Set<Component> mappingTransformationComponentFunctionComponents = mappingTransformationComponentFunction.getComponents(); final Component sqlMapLookup = mappingTransformationComponentFunctionComponents.iterator().next(); final Map<String, String> sqlMapLookupParameterMappings = sqlMapLookup.getParameterMappings(); final String user = readManuallyFromTypeSafeConfig("dswarm.db.metadata.username"); final String pass = readManuallyFromTypeSafeConfig("dswarm.db.metadata.password"); final String db = readManuallyFromTypeSafeConfig("dswarm.db.metadata.schema"); setSqlMapParameterMappings(sqlMapLookupParameterMappings, user, pass, db); testXMLTaskWithTuples("sqlmap.lookup.task.result.json", "sqlmap.mabxml.tuples.json", task); }
public MonitoringHelper startExecution(final Task task) { final EntityIdentification identification = EntityIdentification.of(task); final MDCCloseable mdc = identification.putMDC(); task.getJob().getMappings().forEach(this::monitorEntity); monitorEntity(task.getInputDataModel(), "source"); monitorEntity(task.getOutputDataModel(), "target"); return startMonitoring(executionsTimer, identification, mdc); }
@Test public void testSqlMapSTask() throws Exception { final Task task = getTask("dd-1386.task.json"); final Job job = task.getJob(); final Set<Mapping> mappings = job.getMappings(); final Mapping mapping = mappings.iterator().next(); final Component mappingTransformationComponent = mapping.getTransformation(); final Transformation mappingTransformationComponentFunction = (Transformation) mappingTransformationComponent.getFunction(); final Set<Component> mappingTransformationComponentFunctionComponents = mappingTransformationComponentFunction.getComponents(); final Iterator<Component> iterator = mappingTransformationComponentFunctionComponents.iterator(); final String user = readManuallyFromTypeSafeConfig("dswarm.db.metadata.username"); final String pass = readManuallyFromTypeSafeConfig("dswarm.db.metadata.password"); final String db = readManuallyFromTypeSafeConfig("dswarm.db.metadata.schema"); while(iterator.hasNext()) { final Component component = iterator.next(); if("sqlmap".equals(component.getFunction().getName())) { final Map<String, String> sqlMapLookupParameterMappings = component.getParameterMappings(); setSqlMapParameterMappings(sqlMapLookupParameterMappings, user, pass, db); } } testXMLTaskWithTuples("dd-1386.task.result.json", "dd-1386.marcxml.tuples.json", task); }