@Test public void testEnvelopeXQY() throws IOException { //testing xqy JSON canonical instance scaffolding.createFlow(ENTITY, "testharmonize-xqy-json", FlowType.HARMONIZE, CodeFormat.XQUERY, DataFormat.JSON, false); Files.copy(getResourceStream("flow-runner-test/collector2.xqy"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/collector.xqy"), StandardCopyOption.REPLACE_EXISTING); Files.copy(getResourceStream("flow-runner-test/content-testing-envelope.xqy"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-xqy-json/content.xqy"), StandardCopyOption.REPLACE_EXISTING); installUserModules(getHubAdminConfig(), false); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-xqy-json", FlowType.HARMONIZE); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); JsonNode jsonEnvelope = finalDocMgr.read("2.json").next().getContent(new JacksonHandle()).get(); try { logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope)); } catch (JsonProcessingException e) { e.printStackTrace(); } assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments")); assertNotNull(jsonEnvelope.get("envelope").get("attachments")); }
@Test public void testEnvelopeSJS() throws IOException { scaffolding.createFlow(ENTITY, "testharmonize-sjs-json", FlowType.HARMONIZE, CodeFormat.JAVASCRIPT, DataFormat.JSON, false); //testing sjs JSON canonical instance Files.copy(getResourceStream("flow-runner-test/collector.sjs"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/collector.sjs"), StandardCopyOption.REPLACE_EXISTING); Files.copy(getResourceStream("flow-runner-test/contentTestingEnvelope.sjs"), projectDir.resolve("plugins/entities/" + ENTITY + "/harmonize/testharmonize-sjs-json/content.sjs"), StandardCopyOption.REPLACE_EXISTING); installUserModules(getHubAdminConfig(), false); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize-sjs-json", FlowType.HARMONIZE); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); JsonNode jsonEnvelope = finalDocMgr.read("1.json").next().getContent(new JacksonHandle()).get(); try { logger.debug(new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(jsonEnvelope)); } catch (JsonProcessingException e) { e.printStackTrace(); } assertNull(jsonEnvelope.get("envelope").get("instance").get("Person").get("$attachments")); assertNotNull(jsonEnvelope.get("envelope").get("attachments")); }
@Test public void runJSONErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSONError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); System.out.println(node.asText()); assertEquals(1, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); }
@Test public void runCollector() { assertEquals(0, getStagingDocCount()); assertEquals(0, getFinalDocCount()); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", FlowType.HARMONIZE); HashMap<String, Object> options = new HashMap<>(); // a sneaky attempt to test passing options. this value makes the collector work. options.put("returnStuff", true); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1) .withOptions(options) .withStopOnFailure(true); JobTicket ticket = flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); JsonNode node = jobDocMgr.read("/jobs/" + ticket.getJobId() + ".json").next().getContent(new JacksonHandle()).get(); assertEquals(ticket.getJobId(), node.get("jobId").asText()); assertEquals(0, node.get("successfulEvents").asInt()); assertEquals(0, node.get("failedEvents").asInt()); assertEquals(0, node.get("failedBatches").asInt()); assertEquals("FAILED", node.get("status").asText()); } }
@Test public void runSjsJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalSjsJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_SJS, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
@Test public void runJSONWriterErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSONWriterError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(1, node.get("trace").get("steps").size()); assertEquals("writer", node.get("trace").get("steps").get(0).get("label").asText()); } }
@Test public void runXqyJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.json").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalXqyJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_XQY, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10)
@Test public void runXMLWriterErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXMLWriterError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(1, node.getElementsByTagName("step").getLength()); assertEquals("writer", node.getElementsByTagName("label").item(0).getTextContent()); }
options.put("name", "Bob Smith"); options.put("age", 55); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10)
@Test public void runXMLErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXMLError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(1, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); }
@Test public void runXqyXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.xml").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_XQY, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void runSjsXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); //debugOutput(finalDoc); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_SJS, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
options.put("age", 55); getHubFlowRunnerConfig(); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10)
@Test public void runJSONFlowSansTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSON"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(0, getTracingDocCount()); }
@Test public void runXMLFlowWithTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXML"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); }
@Test public void runJSONFlowWithTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSON"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); }
@Test public void runXMLFlowSansTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXML"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(0, getTracingDocCount()); // disable must be idempotent disableTracing(); disableDebugging(); disableDebugging(); }