int getNumDocsToBeProcessed() { if (getNumDocsCalled) { return 0; } getNumDocsCalled = true; return getDocumentOperations().size(); } }
private void split(Processing processing, Array<Document> innerDocuments) { processing.setVariable(contextFieldName, processing.getDocumentOperations().get(0)); processing.getDocumentOperations().clear(); processing.getDocumentOperations().addAll(innerDocuments.stream() .map(DocumentPut::new) .collect(Collectors.toList())); innerDocuments.clear(); }
private void schemaMapProcessing(Processing processing) { final List<DocumentOperation> documentOperations = processing.getDocumentOperations(); for (int i = 0; i < documentOperations.size(); i++) { DocumentOperation op = documentOperations.get(i); if (op instanceof DocumentPut) { documentOperations.set(i, configDoc(processor, (DocumentPut) op)); } else if (op instanceof DocumentUpdate) { documentOperations.set(i, configDocUpd(processor, (DocumentUpdate) op)); } } }
@Override public Progress process(Processing processing) { if (processing.getDocumentOperations().size() != 1) { //we were given more than one document, return log.log(LogLevel.DEBUG, "More than one document given, returning. (Was given " + processing.getDocumentOperations().size() + " documents)."); return Progress.DONE; } if (!doProcessOuterDocument(processing.getDocumentOperations().get(0), documentTypeName)) { return Progress.DONE; } Document outerDoc = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument();; @SuppressWarnings("unchecked") Array<Document> innerDocuments = (Array<Document>) outerDoc.getFieldValue(arrayFieldName); if (innerDocuments == null) { //the document does not have the field, return log.log(LogLevel.DEBUG, "The given Document does not have a field value for field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } if (innerDocuments.size() == 0) { //the array is empty, return log.log(LogLevel.DEBUG, "The given Document does not have any elements in array field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } split(processing, innerDocuments); return Progress.DONE; }
private void logProgress(Processing processing, DocumentProcessor.Progress progress, Call call) { StringBuilder message = new StringBuilder(); boolean first = true; message.append(call.getDocumentProcessorId()).append(" of class ") .append(call.getDocumentProcessor().getClass().getSimpleName()).append(" returned ").append(progress) .append(" for the documents: ["); for (DocumentOperation op : processing.getDocumentOperations()) { if (first) { first = false; } else { message.append(", "); } if (op instanceof DocumentPut) { message.append(Utf8.toString(JsonWriter.toByteArray(((DocumentPut) op).getDocument()))); } else { message.append(op.toString()); } } message.append("]"); log.log(LogLevel.SPAM, message.toString()); }
final int initialSize = processing.getDocumentOperations().size(); for (DocumentOperation op : processing.getDocumentOperations()) { try { if (op instanceof DocumentPut) {
private void unwrapSchemaMapping(Processing processing) { final List<DocumentOperation> documentOperations = processing.getDocumentOperations(); for (int i = 0; i < documentOperations.size(); i++) { DocumentOperation documentOperation = documentOperations.get(i); if (documentOperation instanceof DocumentPut) { DocumentPut putOperation = (DocumentPut) documentOperation; if (putOperation.getDocument() instanceof DocumentOperationWrapper) { DocumentOperationWrapper proxy = (DocumentOperationWrapper) putOperation.getDocument(); documentOperations.set(i, new DocumentPut(putOperation, ((DocumentPut)proxy.getWrappedDocumentOperation()).getDocument())); } } } }
@Override public void processingDone(List<Processing> processings) { List<DocumentMessage> messages = new ArrayList<>(); if (messageFactory != null) { for (Processing processing : processings) { for (DocumentOperation documentOperation : processing.getDocumentOperations()) { messages.add(messageFactory.fromDocumentOperation(processing, documentOperation)); } } } if (log.isLoggable(LogLevel.DEBUG)) { log.log(LogLevel.DEBUG, "Forwarding " + messages.size() + " messages from " + processings.size() + " processings."); } if (messages.isEmpty()) { dispatchResponse(Response.Status.OK); return; } long inputSequenceId = requestMsg.getSequenceId(); ResponseMerger responseHandler = new ResponseMerger(requestMsg, messages.size(), this); for (Message message : messages) { // See comment for internalNoThrottledSource. dispatchRequest(message, (inputSequenceId == message.getSequenceId()) ? getUri().getPath() : "/" + internalNoThrottledSource, responseHandler); } }
@Override public Progress process(Processing processing) { if ( ! doProcessOuterDocument(processing.getVariable(contextFieldName), documentTypeName)) { return Progress.DONE; } DocumentPut outerDoc = (DocumentPut)processing.getVariable(contextFieldName); @SuppressWarnings("unchecked") Array<Document> innerDocuments = (Array<Document>) outerDoc.getDocument().getFieldValue(arrayFieldName); if (innerDocuments == null) { @SuppressWarnings("unchecked") Array<Document> empty = (Array<Document>) outerDoc.getDocument().getDataType().getField(arrayFieldName).getDataType().createFieldValue(); innerDocuments = empty; } for (DocumentOperation op : processing.getDocumentOperations()) { if (op instanceof DocumentPut) { innerDocuments.add(((DocumentPut)op).getDocument()); } else { log.log(LogLevel.DEBUG, "Skipping: " + op); } } processing.getDocumentOperations().clear(); processing.getDocumentOperations().add(outerDoc); processing.removeVariable(contextFieldName); return Progress.DONE; }
/** * Call the DocumentProcessor of this call. * * @param processing the Processing object to use * @return the progress of the DocumentProcessor that was called */ public DocumentProcessor.Progress call(Processing processing) { try { int numDocs = processing.getDocumentOperations().size(); schemaMapProcessing(processing); long startTime = SystemTimer.INSTANCE.milliTime(); DocumentProcessor.Progress retval = processor.process(processing); incrementProcTime(SystemTimer.INSTANCE.milliTime() - startTime); incrementDocs(numDocs); return retval; } finally { unwrapSchemaMapping(processing); } }