@Test public void testWriteMultipleJSONDocsWithDefaultMetadata() throws KeyManagementException, NoSuchAlgorithmException, Exception { String docId[] = { "/a.json", "/b.json", "/c.json" }; String json1 = new String("{\"animal\":\"dog\", \"says\":\"woof\"}"); String json2 = new String("{\"animal\":\"cat\", \"says\":\"meow\"}"); String json3 = new String("{\"animal\":\"rat\", \"says\":\"keek\"}"); Reader strReader = new StringReader(json1); JSONDocumentManager docMgr = client.newJSONDocumentManager(); DocumentWriteSet writeset = docMgr.newWriteSet(); // put metadata DocumentMetadataHandle mh = setMetadata(); writeset.addDefault(mh); writeset.add(docId[0], new ReaderHandle(strReader).withFormat(Format.JSON)); writeset.add(docId[1], new ReaderHandle(new StringReader(json2))); writeset.add(docId[2], new ReaderHandle(new StringReader(json3))); docMgr.write(writeset); DocumentPage page = docMgr.read(docId); while (page.hasNext()) { DocumentRecord rec = page.next(); docMgr.readMetadata(rec.getUri(), mh); System.out.println(rec.getUri()); validateMetadata(mh); } validateMetadata(mh); }
@Override public void handleBatch(DatabaseClient client, List<? extends DocumentWriteOperation> items) { DocumentManager<?, ?> mgr = buildDocumentManager(client); if (contentFormat != null) { mgr.setContentFormat(contentFormat); } DocumentWriteSet set = mgr.newWriteSet(); for (DocumentWriteOperation item : items) { set.add(item); } int count = set.size(); if (logger.isDebugEnabled()) { logger.debug("Writing " + count + " documents to MarkLogic"); } if (serverTransform != null) { mgr.write(set, serverTransform); } else { mgr.write(set); } if (logger.isInfoEnabled()) { logger.info("Wrote " + count + " documents to MarkLogic"); } }
public void cleanupDocs(DocumentWriteSet docs) { client.newDocumentManager().delete(docs.stream().map(doc -> doc.getUri()).toArray(String[]::new)); }
@Override public void write(List<? extends String> items) throws Exception { DocumentWriteSet set = mgr.newWriteSet(); int size = items.size(); logger.info("Building set of documents to write"); for (int i = 0; i < size; i++) { String xml = items.get(i); String uri = generateUri(xml, i + 1 + ""); set.add(uri, buildMetadata(), new StringHandle(xml)); } logger.info("Writing set of documents, size: " + size); mgr.write(set); logger.info("Finished writing set of documents"); }
mh.with(constructJSONPropertiesMetadata(p)); writeset.addDefault(mh); // Adding default metadata to the entire batch File file1 = null; file1 = new File("src/test/java/com/marklogic/client/functionaltest/data/" + docId[0]); handle1.setFormat(Format.BINARY); writeset.add("/generic/Sega.jpg", handle1); // This document implicitly gets jh.setFormat(Format.JSON); writeset.add("/generic/dog.json", new JacksonHandle().with(mapper.readTree("{\"quality\": 10}")), jh); // This writeset.disableDefault(); String foo1 = "This is foo1 of byte Array"; byte[] ba = foo1.getBytes(); bh.setFormat(Format.TEXT); writeset.add("/generic/foo1.txt", bh); sh.set(ds); sh.setFormat(Format.XML); writeset.add("/generic/foo.xml", new JacksonHandle().with(constructJSONCollectionMetadata("http://Json-Uri-generic-collections/")), sh); // This
public BatchWriter(BatchWriteSet writeSet) { if ( writeSet.getWriteSet().size() == 0 ) { throw new IllegalStateException("Attempt to write an empty batch"); } this.writeSet = writeSet; }
@Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { String currentDir = dir.normalize().toAbsolutePath().toString(); if (isMappingDir(dir, mappingPath.toAbsolutePath())) { Modules modules = new MappingDefModulesFinder().findModules(dir.toString()); DocumentMetadataHandle meta = new DocumentMetadataHandle(); meta.getCollections().add("http://marklogic.com/data-hub/mappings"); documentPermissionsParser.parsePermissions(hubConfig.getModulePermissions(), meta.getPermissions()); for (Resource r : modules.getAssets()) { if (forceLoad || modulesManager.hasFileBeenModifiedSinceLastLoaded(r.getFile())) { InputStream inputStream = r.getInputStream(); StringHandle handle = new StringHandle(IOUtils.toString(inputStream)); inputStream.close(); finalMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); stagingMappingDocumentWriteSet.add("/mappings/" + r.getFile().getParentFile().getName() + "/" + r.getFilename(), meta, handle); modulesManager.saveLastLoadedTimestamp(r.getFile(), new Date()); } } return FileVisitResult.CONTINUE; } else { return FileVisitResult.CONTINUE; } }
batch.add("doc1.json", doc1); // system default metadata batch.addDefault(defaultMetadata1); batch.add("doc2.json", doc2); // batch default metadata batch.add("doc3.json", docSpecificMetadata, doc3); batch.add("doc4.json", doc4); // batch default metadata batch.addDefault(defaultMetadata2); batch.add("doc5.json", doc5); // batch default batch.disableDefault(); batch.add("doc6.json", doc6); // system default metadata batch.addDefault(defaultMetadata1); batch.add("doc7.json", doc7); // batch default metadata batch.disableDefault(); batch.add("doc8.json", doc8); // system default metadata
private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { batchWriteSet.setItemsSoFar(itemsSoFar.addAndGet(batchWriteSet.getWriteSet().size())); WriteBatch batch = batchWriteSet.getBatchOfWriteEvents(); for ( WriteBatchListener successListener : successListeners ) { try { successListener.processEvent(batch); } catch (Throwable t) { logger.error("Exception thrown by an onBatchSuccess listener", t); } } }
writeset.addDefault(metadataHandle); writeset.add(uri, handle);
@Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (isFlowPropertiesFile(file) && modulesManager.hasFileBeenModifiedSinceLastLoaded(file.toFile())) { Flow flow = flowManager.getFlowFromProperties(file); StringHandle handle = new StringHandle(flow.serialize()); handle.setFormat(Format.XML); documentWriteSet.add(flow.getFlowDbPath(), handle); modulesManager.saveLastLoadedTimestamp(file.toFile(), new Date()); } return FileVisitResult.CONTINUE; } });
@Override public WriteBatcher add(DocumentWriteOperation writeOperation) { if ( writeOperation.getUri() == null ) throw new IllegalArgumentException("uri must not be null"); if ( writeOperation.getContent() == null ) throw new IllegalArgumentException("contentHandle must not be null"); initialize(); requireNotStopped(); queue.add(writeOperation); logger.trace("add uri={}", writeOperation.getUri()); // if we have queued batchSize, it's time to flush a batch long recordNum = batchCounter.incrementAndGet(); boolean timeToWriteBatch = (recordNum % getBatchSize()) == 0; if ( timeToWriteBatch ) { BatchWriteSet writeSet = newBatchWriteSet(false); int i=0; for ( ; i < getBatchSize(); i++ ) { DocumentWriteOperation doc = queue.poll(); if ( doc != null ) { writeSet.getWriteSet().add(doc); } else { // strange, there should have been a full batch of docs in the queue... break; } } if ( writeSet.getWriteSet().size() > 0 ) { threadPool.submit( new BatchWriter(writeSet) ); } } return this; }
@Override public void finishBatch() { if ( writeSet.size() > 0 ) { docMgr.write(writeSet); // while this test is usually just 10 records so no more than one write set, // we're ready to do more batches if we want to do performance testing here writeSet = docMgr.newWriteSet(); } } @Override
public WriteBatch getBatchOfWriteEvents() { WriteBatchImpl batch = new WriteBatchImpl() .withBatcher(batcher) .withClient(client) .withJobBatchNumber(batchNumber) .withJobWritesSoFar(itemsSoFar) .withJobTicket(batcher.getJobTicket()); WriteEvent[] writeEvents = getWriteSet().stream() .map(writeOperation -> new WriteEventImpl() .withTargetUri(writeOperation.getUri()) .withContent(writeOperation.getContent()) .withMetadata(writeOperation.getMetadata()) ) .toArray(WriteEventImpl[]::new); batch.withItems(writeEvents); return batch; } }
writeset.addDefault(metadataHandle); writeset.add(uri, handle);
@Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (isFlowPropertiesFile(file) && modulesManager.hasFileBeenModifiedSinceLastLoaded(file.toFile())) { Flow flow = flowManager.getFlowFromProperties(file); StringHandle handle = new StringHandle(flow.serialize()); handle.setFormat(Format.XML); documentWriteSet.add(flow.getFlowDbPath(), handle); modulesManager.saveLastLoadedTimestamp(file.toFile(), new Date()); } return FileVisitResult.CONTINUE; } });
writeset.addDefault(metadataHandle); writeset.add(uri, handle);
@Test public void testWriteValidAndInvalidDocsAndThrowException() { String directory = "/WriteBatcherTest/testWriteValidAndInvalidDocsAndThrowException/"; DocumentWriteSet writeSet = client.newDocumentManager().newWriteSet(); writeSet.add(directory + uri1, new StringHandle("test")); writeSet.add(directory + uri5, new StringHandle("this is not valid XML").withFormat(Format.XML)); writeSet.add(directory + uri2, new StringHandle("test")); writeSet.add(directory + uri5, new StringHandle("this is not valid XML").withFormat(Format.XML)); writeSet.add(directory + uri3, new StringHandle("test")); testExceptions(writeSet, 3, 2); }