@Override public boolean completeEquals(final Object obj) { return Job.class.isInstance(obj) && super.completeEquals(obj) && DMPPersistenceUtil.getMappingUtils().completeEquals(((Job) obj).getMappings(), getMappings()) && DMPPersistenceUtil.getFilterUtils().completeEquals(((Job) obj).getSkipFilter(), getSkipFilter()); } }
public Job createJob() { final Set<Mapping> mappings = Sets.newLinkedHashSet(); final Mapping simpleMapping = simpleMapping(); final Mapping complexMapping = complexMapping(); mappings.add(simpleMapping); mappings.add(complexMapping); final String jobUUID = UUIDService.getUUID(Job.class.getSimpleName()); final Job job = new Job(jobUUID); job.setName("my job"); job.setDescription("my job description"); job.setMappings(mappings); String json = null; try { json = objectMapper.writeValueAsString(job); } catch (final JsonProcessingException e) { e.printStackTrace(); } TaskTest.LOG.debug("job json: " + json); return job; }
@Override public FilterMorphScriptBuilder apply(final Task task) throws DMPConverterException { final Optional<Filter> optionalSkipFilter = Optional.ofNullable(task.getJob().getSkipFilter()); if (!optionalSkipFilter.isPresent()) { LOG.debug("there is no skip filter at job '{}' of task '{}'", task.getJob().getUuid(), task.getUuid()); return this; } final Filter skipFilter = optionalSkipFilter.get(); final Optional<String> optionalSkipFilterExpression = Optional.ofNullable(skipFilter.getExpression()); if (!optionalSkipFilterExpression.isPresent()) { LOG.debug("there is no filter expression in skip filter '{}' of job '{}' of task '{}'", skipFilter.getUuid(), task.getJob().getUuid(), task.getUuid()); return this; } final String filterExpressionString = optionalSkipFilterExpression.get(); final Map<String, FilterExpression> filterExpressionMap = extractFilterExpressions(filterExpressionString); if (filterExpressionMap == null || filterExpressionMap.isEmpty()) { LOG.debug("there are no filter conditions in filter expression in skip filter '{}' of job '{}' of task '{}'", skipFilter.getUuid(), task.getJob().getUuid(), task.getUuid()); return this; } super.apply(task); metaName.setTextContent(SKIP_FILTER + task.getJob().getUuid()); final String attributePathStringXMLEscaped = Iterators.getLast(filterExpressionMap.keySet().iterator()); addFilter(attributePathStringXMLEscaped, COMBINE_VALUE_VARIABLE, filterExpressionMap, rules, false); return this; }
@Override public MorphScriptBuilder apply(final Task task) throws DMPConverterException { super.apply(task); final List<String> metas = new ArrayList<>(); final Optional<Map<String, Integer>> optionalInputSchemaMap = generateInputSchemaMap(task); final Optional<DataModel> optionalInputDataModel = Optional.ofNullable(task) .flatMap(task2 -> Optional.ofNullable(task2.getInputDataModel())); final Optional<Schema> optionalInputSchema = optionalInputDataModel .flatMap(inputDataModel -> Optional.ofNullable(inputDataModel.getSchema())); final boolean isXmlSchema = isXmlSchema(optionalInputDataModel); final List<Tuple5<Optional<String>, javaslang.collection.List<String>, String, Element, Optional<String>>> mappingOutputs = new ArrayList<>(); for (final Mapping mapping : task.getJob().getMappings()) { metas.add(MorphScriptBuilder.MAPPING_PREFIX + mapping.getUuid()); final Optional<String> optionalDeepestMappingInput = determineDeepestMappingInputAttributePath(mapping, optionalInputSchemaMap); final Optional<Boolean> optionalSelectValueFromSameSubEntity = determineFilterUseCase(mapping, optionalInputSchema); final Optional<String> optionalCommonAttributePathOfMappingInputs = determineCommonAttributePathOfMappingInputs(mapping, optionalInputSchema, isXmlSchema); createTransformation(rules, mapping, optionalDeepestMappingInput, optionalSelectValueFromSameSubEntity, optionalCommonAttributePathOfMappingInputs) .ifPresent(currentMappingOutputs -> currentMappingOutputs .forEach(currentMappingOutput -> mappingOutputs.add(Tuple.of(currentMappingOutput._1, currentMappingOutput._2, currentMappingOutput._3, currentMappingOutput._4, optionalCommonAttributePathOfMappingInputs)))); createLookupTable(maps, mapping.getTransformation()); } createMappingOutputs(mappingOutputs, rules, doc); metaName.setTextContent(Joiner.on(", ").join(metas)); return this; }
@Test public void simpleJobTest() { final Set<Mapping> mappings = Sets.newLinkedHashSet(); final Mapping simpleMapping = simpleMapping(); final Mapping complexMapping = complexMapping(); mappings.add(simpleMapping); mappings.add(complexMapping); final String uuid = UUIDService.getUUID(Job.class.getSimpleName()); final Job job = new Job(uuid); job.setMappings(mappings); String json = null; try { json = objectMapper.writeValueAsString(job); } catch (final JsonProcessingException e) { e.printStackTrace(); } JobTest.LOG.debug("job json: " + json); }
public MonitoringHelper startExecution(final Task task) { final EntityIdentification identification = EntityIdentification.of(task); final MDCCloseable mdc = identification.putMDC(); task.getJob().getMappings().forEach(this::monitorEntity); monitorEntity(task.getInputDataModel(), "source"); monitorEntity(task.getOutputDataModel(), "target"); return startMonitoring(executionsTimer, identification, mdc); }
@Test public void testSqlDbRequestTask() throws Exception { final Task task = getTask("sqldbrequest.task.json"); final Job job = task.getJob(); final Set<Mapping> mappings = job.getMappings(); final Mapping mapping = mappings.iterator().next(); final Component mappingTransformationComponent = mapping.getTransformation(); final Transformation mappingTransformationComponentFunction = (Transformation) mappingTransformationComponent.getFunction(); final Set<Component> mappingTransformationComponentFunctionComponents = mappingTransformationComponentFunction.getComponents(); final Component sqlMapLookup = mappingTransformationComponentFunctionComponents.iterator().next(); final Map<String, String> sqlMapLookupParameterMappings = sqlMapLookup.getParameterMappings(); final String user = readManuallyFromTypeSafeConfig("dswarm.db.metadata.username"); final String pass = readManuallyFromTypeSafeConfig("dswarm.db.metadata.password"); final String db = readManuallyFromTypeSafeConfig("dswarm.db.metadata.schema"); setSqlMapParameterMappings(sqlMapLookupParameterMappings, user, pass, db); testXMLTaskWithTuples("sqldbrequest.task.result.json", "sqldbrequest.mabxml.tuples.json", task); }
@Test public void testSqlMapSTask() throws Exception { final Task task = getTask("dd-1386.task.json"); final Job job = task.getJob(); final Set<Mapping> mappings = job.getMappings(); final Mapping mapping = mappings.iterator().next(); final Component mappingTransformationComponent = mapping.getTransformation(); final Transformation mappingTransformationComponentFunction = (Transformation) mappingTransformationComponent.getFunction(); final Set<Component> mappingTransformationComponentFunctionComponents = mappingTransformationComponentFunction.getComponents(); final Iterator<Component> iterator = mappingTransformationComponentFunctionComponents.iterator(); final String user = readManuallyFromTypeSafeConfig("dswarm.db.metadata.username"); final String pass = readManuallyFromTypeSafeConfig("dswarm.db.metadata.password"); final String db = readManuallyFromTypeSafeConfig("dswarm.db.metadata.schema"); while(iterator.hasNext()) { final Component component = iterator.next(); if("sqlmap".equals(component.getFunction().getName())) { final Map<String, String> sqlMapLookupParameterMappings = component.getParameterMappings(); setSqlMapParameterMappings(sqlMapLookupParameterMappings, user, pass, db); } } testXMLTaskWithTuples("dd-1386.task.result.json", "dd-1386.marcxml.tuples.json", task); }
@Test public void testSqlMapTask() throws Exception { final Task task = getTask("sqlmap.lookup.task.json"); final Job job = task.getJob(); final Set<Mapping> mappings = job.getMappings(); final Mapping mapping = mappings.iterator().next(); final Component mappingTransformationComponent = mapping.getTransformation(); final Transformation mappingTransformationComponentFunction = (Transformation) mappingTransformationComponent.getFunction(); final Set<Component> mappingTransformationComponentFunctionComponents = mappingTransformationComponentFunction.getComponents(); final Component sqlMapLookup = mappingTransformationComponentFunctionComponents.iterator().next(); final Map<String, String> sqlMapLookupParameterMappings = sqlMapLookup.getParameterMappings(); final String user = readManuallyFromTypeSafeConfig("dswarm.db.metadata.username"); final String pass = readManuallyFromTypeSafeConfig("dswarm.db.metadata.password"); final String db = readManuallyFromTypeSafeConfig("dswarm.db.metadata.schema"); setSqlMapParameterMappings(sqlMapLookupParameterMappings, user, pass, db); testXMLTaskWithTuples("sqlmap.lookup.task.result.json", "sqlmap.mabxml.tuples.json", task); }