@Override public <T extends AbstractReadHandle> T nextContent(T contentHandle) { return next().getContent(contentHandle); }
protected void writeDocumentToFile(DocumentRecord documentRecord, File file) throws IOException { file.getParentFile().mkdirs(); FileOutputStream fos = new FileOutputStream(file); try { InputStream in = documentRecord.getContent(new InputStreamHandle()).get(); byte[] buffer = new byte[4096]; int bytesRead = -1; while ((bytesRead = in.read(buffer)) != -1) { fos.write(buffer, 0, bytesRead); } fos.flush(); } finally { fos.close(); } }
@Override public void accept(DocumentRecord documentRecord) { String uri = documentRecord.getUri(); ZipEntry zipEntry = buildZipEntry(documentRecord); synchronized (this.zipOutputStream) { try { zipOutputStream.putNextEntry(zipEntry); if (logger.isDebugEnabled()) { logger.debug("Writing zip entry, name: " + zipEntry.getName()); } zipOutputStream.write(documentRecord.getContent(new BytesHandle()).get()); zipOutputStream.closeEntry(); } catch (IOException e) { throw new RuntimeException("Unable to write zip entry for URI: " + uri + "; cause: " + e.getMessage(), e); } } }
@Override public String generateOutput(DocumentRecord documentRecord) { if (Format.XML.equals(documentRecord.getFormat())) { DOMHandle handle = documentRecord.getContent(new DOMHandle()); Document document = handle.get(); OutputFormat format = new OutputFormat(handle.get()); format.setOmitXMLDeclaration(omitXmlDeclaration); StringWriter writer = new StringWriter(); try { new XMLSerializer(writer, format).serialize(document); return writer.toString(); } catch (IOException e) { throw new RuntimeException("Unable to serialize XML document to string: " + e.getMessage(), e); } } else if (logger.isDebugEnabled()) { logger.debug(String.format("Document '%s' has a format of '%s', so will not attempt to remove the XML declaration from it", documentRecord.getUri(), documentRecord.getFormat().name())); } return documentRecord.getContent(new StringHandle()).get(); }
protected String getModulesFile(String uri) { try { String contents = modMgr.read(uri).next().getContent(new StringHandle()).get(); return contents.replaceFirst("(\\(:|//)\\s+cache\\sbuster:.+\\n", ""); } catch (IllegalStateException e){ return null; } catch(Exception e) { e.printStackTrace(); } return null; }
protected Document getModulesDocument(String uri) { return modMgr.read(uri).next().getContent(new DOMHandle()).get(); }
@Test public void runJSONErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSONError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); System.out.println(node.asText()); assertEquals(1, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); }
@Test public void runSjsJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalSjsJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_SJS, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
@Test public void runXqyJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.json").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalXqyJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_XQY, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
@Test public void runCollector() { assertEquals(0, getStagingDocCount()); assertEquals(0, getFinalDocCount()); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", FlowType.HARMONIZE); HashMap<String, Object> options = new HashMap<>(); // a sneaky attempt to test passing options. this value makes the collector work. options.put("returnStuff", true); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1) .withOptions(options) .withStopOnFailure(true); JobTicket ticket = flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); JsonNode node = jobDocMgr.read("/jobs/" + ticket.getJobId() + ".json").next().getContent(new JacksonHandle()).get(); assertEquals(ticket.getJobId(), node.get("jobId").asText()); assertEquals(0, node.get("successfulEvents").asInt()); assertEquals(0, node.get("failedEvents").asInt()); assertEquals(0, node.get("failedBatches").asInt()); assertEquals("FAILED", node.get("status").asText()); } }
@Test public void runJSONWriterErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSONWriterError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(1, node.get("trace").get("steps").size()); assertEquals("writer", node.get("trace").get("steps").get(0).get("label").asText()); } }
@Test public void runXqyXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.xml").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_XQY, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void testRunFlowWithAll() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); HashMap<String, String> modules = new HashMap<>(); modules.put("/entities/test/harmonize/my-test-flow-with-all/my-test-flow-with-all.xml", "flow-manager-test/my-test-flow-with-all/my-test-flow-with-all.xml"); modules.put("/entities/test/harmonize/my-test-flow-with-all/collector.xqy", "flow-manager-test/my-test-flow-with-all/collector.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/headers.xqy", "flow-manager-test/my-test-flow-with-all/headers.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/content.xqy", "flow-manager-test/my-test-flow-with-all/content.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/triples.xqy", "flow-manager-test/my-test-flow-with-all/triples.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/writer.xqy", "flow-manager-test/my-test-flow-with-all/writer.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-all/main.xqy", "flow-manager-test/my-test-flow-with-all/main.xqy"); installModules(modules); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow-with-all"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-all/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-all/\")"); }
@Test public void testRunFlowWithHeader() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); HashMap<String, String> modules = new HashMap<>(); modules.put("/entities/test/harmonize/my-test-flow-with-header/flow.xml", "flow-manager-test/my-test-flow-with-header/flow.xml"); modules.put("/entities/test/harmonize/my-test-flow-with-header/collector.xqy", "flow-manager-test/my-test-flow-with-header/collector.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/content.xqy", "flow-manager-test/my-test-flow-with-header/content.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/headers.xqy", "flow-manager-test/my-test-flow-with-header/headers.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/triples.xqy", "flow-manager-test/my-test-flow-with-header/triples.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/writer.xqy", "flow-manager-test/my-test-flow-with-header/writer.xqy"); modules.put("/entities/test/harmonize/my-test-flow-with-header/main.xqy", "flow-manager-test/my-test-flow-with-header/main.xqy"); installModules(modules); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow-with-header"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized-with-header/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); runInModules("xdmp:directory-delete(\"/entities/test/harmonize/my-test-flow-with-header/\")"); }
@Test public void runSjsXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); //debugOutput(finalDoc); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_SJS, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void runXMLWriterErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXMLWriterError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(1, node.getElementsByTagName("step").getLength()); assertEquals("writer", node.getElementsByTagName("label").item(0).getTextContent()); }
@Test public void runXMLErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXMLError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(1, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); }
@Test public void testProjectScaffolding() throws IOException { DatabaseClient stagingTriggersClient = null; DatabaseClient finalTriggersClient = null; DatabaseClient stagingSchemasClient = null; DatabaseClient finalSchemasClient = null; try { stagingTriggersClient = getClient(host, stagingPort, HubConfig.DEFAULT_STAGING_TRIGGERS_DB_NAME, user, password, stagingAuthMethod); finalTriggersClient = getClient(host, finalPort, HubConfig.DEFAULT_FINAL_TRIGGERS_DB_NAME, user, password, finalAuthMethod); stagingSchemasClient = getClient(host, stagingPort, HubConfig.DEFAULT_STAGING_SCHEMAS_DB_NAME, user, password, stagingAuthMethod); finalSchemasClient = getClient(host, finalPort, HubConfig.DEFAULT_FINAL_SCHEMAS_DB_NAME, user, password, finalAuthMethod); } catch (Exception e) { throw new RuntimeException(e); } //checking if modules are written to correct db assertNotNull(getModulesFile("/ext/sample-trigger.xqy")); assertNotNull(getModulesFile("/ext/sample-trigger.xqy")); //checking if tdes are written to correct db Document expectedXml = getXmlFromResource("data-hub-test/scaffolding/tdedoc.xml"); Document actualXml = stagingSchemasClient.newDocumentManager().read("/tde/tdedoc.xml").next().getContent(new DOMHandle()).get(); assertXMLEqual(expectedXml, actualXml); actualXml = finalSchemasClient.newDocumentManager().read("/tde/tdedoc.xml").next().getContent(new DOMHandle()).get(); assertXMLEqual(expectedXml, actualXml); //checking if triggers are written assertTrue(stagingTriggersClient.newServerEval().xquery("fn:count(fn:doc())").eval().next().getNumber().intValue() == 1); assertTrue(finalTriggersClient.newServerEval().xquery("fn:count(fn:doc())").eval().next().getNumber().intValue() == 1); }
@Test public void testRunFlow() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); installModules(); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow1"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); DocumentMetadataHandle metadata = finalDocMgr.readMetadata("/employee1.xml", new DocumentMetadataHandle()); DocumentMetadataHandle.DocumentPermissions permissions = metadata.getPermissions(); assertEquals( permissions.get("harmonized-reader").toString(), "[READ]", "Default permissions on harmonized documents should contain harmonized-reader/read"); assertEquals(permissions.get("harmonized-updater").toString(), "[UPDATE]", "Default permissions on harmonized documents should contain harmonized-updater/update"); }
@Test public void testRunFlowWithBackwards() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addFinalDocs(); installModules(); assertEquals(0, getStagingDocCount()); assertEquals(2, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow1"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1) .withSourceClient(getHubFlowRunnerConfig().newReverseFlowClient()) .withDestinationDatabase(HubConfig.DEFAULT_STAGING_NAME); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), stagingDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), stagingDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); DocumentMetadataHandle metadata = stagingDocMgr.readMetadata("/employee1.xml", new DocumentMetadataHandle()); DocumentMetadataHandle.DocumentPermissions permissions = metadata.getPermissions(); assertEquals(permissions.get("harmonized-reader").toString(), "[READ]", "Default permissions on harmonized documents should contain harmonized-reader/read"); assertEquals(permissions.get("harmonized-updater").toString(), "[UPDATE]", "Default permissions on harmonized documents should contain harmonized-updater/update"); }