public BatchWriter(BatchWriteSet writeSet) { if ( writeSet.getWriteSet().size() == 0 ) { throw new IllegalStateException("Attempt to write an empty batch"); } this.writeSet = writeSet; }
private void sendSuccessToListeners(BatchWriteSet batchWriteSet) { batchWriteSet.setItemsSoFar(itemsSoFar.addAndGet(batchWriteSet.getWriteSet().size())); WriteBatch batch = batchWriteSet.getBatchOfWriteEvents(); for ( WriteBatchListener successListener : successListeners ) { try { successListener.processEvent(batch); } catch (Throwable t) { logger.error("Exception thrown by an onBatchSuccess listener", t); } } }
@Override public void handleBatch(DatabaseClient client, List<? extends DocumentWriteOperation> items) { DocumentManager<?, ?> mgr = buildDocumentManager(client); if (contentFormat != null) { mgr.setContentFormat(contentFormat); } DocumentWriteSet set = mgr.newWriteSet(); for (DocumentWriteOperation item : items) { set.add(item); } int count = set.size(); if (logger.isDebugEnabled()) { logger.debug("Writing " + count + " documents to MarkLogic"); } if (serverTransform != null) { mgr.write(set, serverTransform); } else { mgr.write(set); } if (logger.isInfoEnabled()) { logger.info("Wrote " + count + " documents to MarkLogic"); } }
@Override public void finishBatch() { if ( writeSet.size() > 0 ) { docMgr.write(writeSet); // while this test is usually just 10 records so no more than one write set, // we're ready to do more batches if we want to do performance testing here writeSet = docMgr.newWriteSet(); } } @Override
@Override public WriteBatcher add(DocumentWriteOperation writeOperation) { if ( writeOperation.getUri() == null ) throw new IllegalArgumentException("uri must not be null"); if ( writeOperation.getContent() == null ) throw new IllegalArgumentException("contentHandle must not be null"); initialize(); requireNotStopped(); queue.add(writeOperation); logger.trace("add uri={}", writeOperation.getUri()); // if we have queued batchSize, it's time to flush a batch long recordNum = batchCounter.incrementAndGet(); boolean timeToWriteBatch = (recordNum % getBatchSize()) == 0; if ( timeToWriteBatch ) { BatchWriteSet writeSet = newBatchWriteSet(false); int i=0; for ( ; i < getBatchSize(); i++ ) { DocumentWriteOperation doc = queue.poll(); if ( doc != null ) { writeSet.getWriteSet().add(doc); } else { // strange, there should have been a full batch of docs in the queue... break; } } if ( writeSet.getWriteSet().size() > 0 ) { threadPool.submit( new BatchWriter(writeSet) ); } } return this; }