protected String getModulesFile(String uri) { try { String contents = modMgr.read(uri).next().getContent(new StringHandle()).get(); return contents.replaceFirst("(\\(:|//)\\s+cache\\sbuster:.+\\n", ""); } catch (IllegalStateException e){ return null; } catch(Exception e) { e.printStackTrace(); } return null; }
protected Document getModulesDocument(String uri) { return modMgr.read(uri).next().getContent(new DOMHandle()).get(); }
@Test public void test_issue_758() throws Exception { BinaryDocumentManager docMgr = Common.client.newBinaryDocumentManager(); DocumentWriteSet writeset =docMgr.newWriteSet(); FileHandle h1 = new FileHandle(new File( "../marklogic-client-api-functionaltests/src/test/java/com/marklogic" + "/client" + "/functionaltest/data" + "/Sega-4MB.jpg")); String uri = "BinaryDocumentTest_" + new Random().nextInt(10000) + "/" + "Sega-4MB.jpg"; writeset.add(uri, h1); docMgr.write(writeset); DocumentPage page = docMgr.read(uri); DocumentRecord rec = page.next(); assertNotNull("DocumentRecord should never be null", rec); assertEquals(rec.getFormat(),Format.BINARY); } }
@Test public void testWriteMultipleJSONDocsWithDefaultMetadata() throws KeyManagementException, NoSuchAlgorithmException, Exception { String docId[] = { "/a.json", "/b.json", "/c.json" }; String json1 = new String("{\"animal\":\"dog\", \"says\":\"woof\"}"); String json2 = new String("{\"animal\":\"cat\", \"says\":\"meow\"}"); String json3 = new String("{\"animal\":\"rat\", \"says\":\"keek\"}"); Reader strReader = new StringReader(json1); JSONDocumentManager docMgr = client.newJSONDocumentManager(); DocumentWriteSet writeset = docMgr.newWriteSet(); // put metadata DocumentMetadataHandle mh = setMetadata(); writeset.addDefault(mh); writeset.add(docId[0], new ReaderHandle(strReader).withFormat(Format.JSON)); writeset.add(docId[1], new ReaderHandle(new StringReader(json2))); writeset.add(docId[2], new ReaderHandle(new StringReader(json3))); docMgr.write(writeset); DocumentPage page = docMgr.read(docId); while (page.hasNext()) { DocumentRecord rec = page.next(); docMgr.readMetadata(rec.getUri(), mh); System.out.println(rec.getUri()); validateMetadata(mh); } validateMetadata(mh); }
@Test public void runJSONErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSONError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); System.out.println(node.asText()); assertEquals(1, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); }
@Test public void runSjsJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalSjsJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_SJS, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
@Test public void runXqyJsonFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyJsonWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.json").next(); String finalDoc= doc.getContent(new StringHandle()).get(); assertJsonEqual(getResource("tracing-test/traces/finalXqyJsonDoc.json"), finalDoc, true); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(4, node.get("trace").get("steps").size()); assertEquals("content", node.get("trace").get("steps").get(0).get("label").asText()); assertEquals(BINARY_HEX_ENCODED_XQY, node.get("trace").get("steps").get(0).get("output").asText().toLowerCase()); }
@Test public void runCollector() { assertEquals(0, getStagingDocCount()); assertEquals(0, getFinalDocCount()); Flow harmonizeFlow = fm.getFlow(ENTITY, "testharmonize", FlowType.HARMONIZE); HashMap<String, Object> options = new HashMap<>(); // a sneaky attempt to test passing options. this value makes the collector work. options.put("returnStuff", true); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(harmonizeFlow) .withBatchSize(10) .withThreadCount(1) .withOptions(options) .withStopOnFailure(true); JobTicket ticket = flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); JsonNode node = jobDocMgr.read("/jobs/" + ticket.getJobId() + ".json").next().getContent(new JacksonHandle()).get(); assertEquals(ticket.getJobId(), node.get("jobId").asText()); assertEquals(0, node.get("successfulEvents").asInt()); assertEquals(0, node.get("failedEvents").asInt()); assertEquals(0, node.get("failedBatches").asInt()); assertEquals("FAILED", node.get("status").asText()); } }
@Test public void runJSONWriterErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeJSONWriterError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); JsonNode node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new JacksonHandle()).get(); assertEquals(1, node.get("trace").get("steps").size()); assertEquals("writer", node.get("trace").get("steps").get(0).get("label").asText()); } }
@Test public void runXqyXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXqyXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("/doc/1.xml").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_XQY, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void runSjsXmlFlowWithBinaryContent() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); enableTracing(); assertTrue(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeSjsXmlWithBinary"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(5, getFinalDocCount()); assertEquals(6, getTracingDocCount()); DocumentRecord doc = finalDocMgr.read("1").next(); Document finalDoc = doc.getContent(new DOMHandle()).get(); //debugOutput(finalDoc); assertXMLEqual(getXmlFromResource("tracing-test/traces/finalSjsXmlDoc.xml"), finalDoc); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(4, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); assertEquals(BINARY_HEX_ENCODED_SJS, node.getElementsByTagName("output").item(0).getTextContent().toLowerCase()); }
@Test public void runXMLWriterErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXMLWriterError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(1, node.getElementsByTagName("step").getLength()); assertEquals("writer", node.getElementsByTagName("label").item(0).getTextContent()); }
@Test public void runXMLErrorFlowWithoutTracing() { assertEquals(0, getFinalDocCount()); assertEquals(0, getTracingDocCount()); Tracing t = Tracing.create(flowRunnerClient); assertFalse(t.isEnabled()); Flow flow = fm.getFlow("trace-entity", "tracemeXMLError"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); assertEquals(0, getFinalDocCount()); assertEquals(5, getTracingDocCount()); Document node = jobDocMgr.search(allButCollectors(), 1).next().getContent(new DOMHandle()).get(); assertEquals(1, node.getElementsByTagName("step").getLength()); assertEquals("content", node.getElementsByTagName("label").item(0).getTextContent()); }
@Test public void testRunFlow() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addStagingDocs(); installModules(); assertEquals(2, getStagingDocCount()); assertEquals(0, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow1"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), finalDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), finalDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); DocumentMetadataHandle metadata = finalDocMgr.readMetadata("/employee1.xml", new DocumentMetadataHandle()); DocumentMetadataHandle.DocumentPermissions permissions = metadata.getPermissions(); assertEquals( permissions.get("harmonized-reader").toString(), "[READ]", "Default permissions on harmonized documents should contain harmonized-reader/read"); assertEquals(permissions.get("harmonized-updater").toString(), "[UPDATE]", "Default permissions on harmonized documents should contain harmonized-updater/update"); }
@Test public void testRunFlowWithBackwards() throws SAXException, IOException, ParserConfigurationException, XMLStreamException { addFinalDocs(); installModules(); assertEquals(0, getStagingDocCount()); assertEquals(2, getFinalDocCount()); getHubFlowRunnerConfig(); Flow flow1 = fm.getFlow("test", "my-test-flow1"); FlowRunner flowRunner = fm.newFlowRunner() .withFlow(flow1) .withBatchSize(10) .withThreadCount(1) .withSourceClient(getHubFlowRunnerConfig().newReverseFlowClient()) .withDestinationDatabase(HubConfig.DEFAULT_STAGING_NAME); flowRunner.run(); flowRunner.awaitCompletion(); getHubAdminConfig(); assertEquals(2, getStagingDocCount()); assertEquals(2, getFinalDocCount()); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized1.xml"), stagingDocMgr.read("/employee1.xml").next().getContent(new DOMHandle()).get() ); assertXMLEqual(getXmlFromResource("flow-manager-test/harmonized/harmonized2.xml"), stagingDocMgr.read("/employee2.xml").next().getContent(new DOMHandle()).get()); DocumentMetadataHandle metadata = stagingDocMgr.readMetadata("/employee1.xml", new DocumentMetadataHandle()); DocumentMetadataHandle.DocumentPermissions permissions = metadata.getPermissions(); assertEquals(permissions.get("harmonized-reader").toString(), "[READ]", "Default permissions on harmonized documents should contain harmonized-reader/read"); assertEquals(permissions.get("harmonized-updater").toString(), "[UPDATE]", "Default permissions on harmonized documents should contain harmonized-updater/update"); }
@Test public void testBulkSearchSQDwithResponseFormatandStringHandle() throws KeyManagementException, NoSuchAlgorithmException, Exception { loadTxtDocuments(); loadJSONDocuments(); TextDocumentManager docMgr = client.newTextDocumentManager(); QueryManager queryMgr = client.newQueryManager(); StringQueryDefinition qd = queryMgr.newStringDefinition(); qd.setCriteria("bar"); docMgr.setNonDocumentFormat(Format.JSON); docMgr.setSearchView(QueryView.METADATA); docMgr.setMetadataCategories(Metadata.PERMISSIONS); StringHandle results = new StringHandle().withFormat(Format.JSON); DocumentPage page = docMgr.search(qd, 1, results); DocumentMetadataHandle mh = new DocumentMetadataHandle(); while (page.hasNext()) { DocumentRecord rec = page.next(); validateRecord(rec, Format.TEXT); docMgr.readMetadata(rec.getUri(), mh); assertTrue("Records has permissions? ", mh.getPermissions().containsKey("flexrep-eval")); assertTrue("Record has collections ?", mh.getCollections().isEmpty()); } assertFalse("Search handle contains", results.get().isEmpty()); }
@Test public void testBulkSearchSQDwithResponseFormatandStringHandle() throws KeyManagementException, NoSuchAlgorithmException, Exception { loadJSONDocuments(); JSONDocumentManager docMgr = client.newJSONDocumentManager(); QueryManager queryMgr = client.newQueryManager(); StructuredQueryBuilder qb = new StructuredQueryBuilder(); StructuredQueryDefinition qd = qb.and(qb.term("dog1", "dog11")); queryMgr.search(qd, new SearchHandle()); docMgr.setNonDocumentFormat(Format.JSON); docMgr.setSearchView(QueryView.METADATA); docMgr.setMetadataCategories(Metadata.PERMISSIONS); StringHandle results = new StringHandle().withFormat(Format.JSON); DocumentPage page = docMgr.search(qd, 1, results); DocumentMetadataHandle mh = new DocumentMetadataHandle(); while (page.hasNext()) { DocumentRecord rec = page.next(); validateRecord(rec, Format.JSON); docMgr.readMetadata(rec.getUri(), mh); assertTrue("Records has permissions? ", mh.getPermissions().containsKey("flexrep-eval")); assertTrue("Record has collections ?", mh.getCollections().isEmpty()); } assertFalse("Search handle contains", results.get().isEmpty()); }
@Test public void testWriteMultipleTextDocWithDefaultMetadata() { String docId[] = { "/foo/test/myFoo1.txt", "/foo/test/myFoo2.txt", "/foo/test/myFoo3.txt" }; TextDocumentManager docMgr = client.newTextDocumentManager(); DocumentWriteSet writeset = docMgr.newWriteSet(); // put metadata DocumentMetadataHandle mh = setMetadata(); writeset.addDefault(mh); writeset.add(docId[0], new StringHandle().with("This is so foo1")); writeset.add(docId[1], new StringHandle().with("This is so foo2")); writeset.add(docId[2], new StringHandle().with("This is so foo3")); docMgr.write(writeset); DocumentPage page = docMgr.read(docId); while (page.hasNext()) { DocumentRecord rec = page.next(); docMgr.readMetadata(rec.getUri(), mh); validateMetadata(mh); } validateMetadata(mh); }
@Test public void testWriteMultipleXMLDocWithDefaultMetadata() throws KeyManagementException, NoSuchAlgorithmException, Exception { String docId[] = { "/foo/test/Foo1.xml", "/foo/test/Foo2.xml", "/foo/test/Foo3.xml" }; XMLDocumentManager docMgr = client.newXMLDocumentManager(); DocumentWriteSet writeset = docMgr.newWriteSet(); // put metadata DocumentMetadataHandle mh = setMetadata(); writeset.addDefault(mh); writeset.add(docId[0], new DOMHandle(getDocumentContent("This is so foo1"))); writeset.add(docId[1], new DOMHandle().with(getDocumentContent("This is so foo2"))); writeset.add(docId[2], new DOMHandle().with(getDocumentContent("This is so foo3"))); docMgr.write(writeset); DocumentPage page = docMgr.read(docId); assertTrue("DocumentPage Size did not return expected value:: returned== " + page.size(), page.size() == 3); while (page.hasNext()) { DocumentRecord rec = page.next(); docMgr.readMetadata(rec.getUri(), mh); validateMetadata(mh); } validateMetadata(mh); }
@Test public void testBulkReadMetadataUsingMultipleUriNoTransaction() throws KeyManagementException, NoSuchAlgorithmException, Exception { String docId[] = { "/foo/test/URIFoo1.txt", "/foo/test/URIFoo2.txt", "/foo/test/URIFoo3.txt" }; DocumentMetadataHandle mhRead = new DocumentMetadataHandle(); TextDocumentManager docMgr = client.newTextDocumentManager(); docMgr.setMetadataCategories(Metadata.ALL); DocumentWriteSet writeset = docMgr.newWriteSet(); // put meta-data DocumentMetadataHandle mh = setMetadata(); writeset.addDefault(mh); writeset.add(docId[0], new StringHandle().with("This is so URI foo 1")); writeset.add(docId[1], new StringHandle().with("This is so URI foo 2")); writeset.add(docId[2], new StringHandle().with("This is so URI foo 3")); docMgr.write(writeset); DocumentPage page = docMgr.readMetadata(docId[0], docId[1], docId[2]); // Issue #294 DocumentPage.size() should return correct size assertTrue("DocumentPage Size did not return expected value:: returned== " + page.size(), page.size() == 3); while (page.hasNext()) { DocumentRecord rec = page.next(); rec.getMetadata(mhRead); validateMetadata(mhRead); } validateMetadata(mhRead); mhRead = null; }