private Processing createProcessing(DocumentOperation documentOperation, Message message) { Processing processing = new Processing(); processing.addDocumentOperation(documentOperation); processing.setServiceName(serviceName); processing.setDocprocServiceRegistry(docprocServiceComponentRegistry); processing.setVariable("route", message.getRoute()); processing.setVariable("timeout", message.getTimeRemaining()); return processing; } }
int getNumDocsToBeProcessed() { if (getNumDocsCalled) { return 0; } getNumDocsCalled = true; return getDocumentOperations().size(); } }
private void processingFailed(Processing processing, String errorMsg, Exception e) { if (e != null) { if (e instanceof HandledProcessingException) { errorMsg += ". Error message: " + e.getMessage(); log.log(Level.WARNING, errorMsg); log.log(Level.FINE, "Chained exception:", e); } else { log.log(Level.WARNING, errorMsg, e); } } else { log.log(Level.WARNING, errorMsg); } //notify endpoint ProcessingEndpoint recv = processing.getEndpoint(); if (recv != null) { recv.processingFailed(processing, e); } }
private void split(Processing processing, Array<Document> innerDocuments) { processing.setVariable(contextFieldName, processing.getDocumentOperations().get(0)); processing.getDocumentOperations().clear(); processing.getDocumentOperations().addAll(innerDocuments.stream() .map(DocumentPut::new) .collect(Collectors.toList())); innerDocuments.clear(); }
@Override public Progress process(Processing processing) { if ( ! doProcessOuterDocument(processing.getVariable(contextFieldName), documentTypeName)) { return Progress.DONE; } DocumentPut outerDoc = (DocumentPut)processing.getVariable(contextFieldName); @SuppressWarnings("unchecked") Array<Document> innerDocuments = (Array<Document>) outerDoc.getDocument().getFieldValue(arrayFieldName); if (innerDocuments == null) { @SuppressWarnings("unchecked") Array<Document> empty = (Array<Document>) outerDoc.getDocument().getDataType().getField(arrayFieldName).getDataType().createFieldValue(); innerDocuments = empty; } for (DocumentOperation op : processing.getDocumentOperations()) { if (op instanceof DocumentPut) { innerDocuments.add(((DocumentPut)op).getDocument()); } else { log.log(LogLevel.DEBUG, "Skipping: " + op); } } processing.getDocumentOperations().clear(); processing.getDocumentOperations().add(outerDoc); processing.removeVariable(contextFieldName); return Progress.DONE; }
processing.setServiceName(getName()); if (processing.callStack() == null) { processing.setCallStack(new CallStack(getCallStack())); Call call = processing.callStack().pop(); if (call == null) { processing.callStack().addNext(call); return progress;
/** * Processes the given Processing through the specified chain. Note that if one * {@link com.yahoo.docproc.DocumentProcessor DocumentProcessor} in the * chain returns a {@link com.yahoo.docproc.DocumentProcessor.LaterProgress DocumentProcessor.LaterProgress}, * it will be returned by this method. This method will hence return whenever a document processor returns any * {@link com.yahoo.docproc.DocumentProcessor.Progress DocumentProcessor.Progress}, or * throws an exception. * * @param chain the specification of the chain to execute * @param processing the Processing to process * @return any Progress * @throws RuntimeException if one of the document processors in the chain throws */ public DocumentProcessor.Progress processOnce(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); return executor.process(processing); }
ProcessingEndpoint recv = processing.getEndpoint(); if (recv != null) { recv.processingDone(processing); processingFailed(processing, processing + " failed at " + processing.callStack().getLastPopped(), null); return true; } else if (DocumentProcessor.Progress.PERMANENT_FAILURE.equals(progress)) { processingFailed(processing, processing + " failed PERMANENTLY at " + processing.callStack().getLastPopped() + ", disabling processing service.", null); setInService(false); return true;
/** * Create a Processing from the given document operation */ public static Processing of(DocumentOperation documentOperation) { return new Processing(documentOperation); }
private static void logProcessingFailure(Processing processing, Exception exception) { //LOGGING ONLY: String errorMsg = processing + " failed at " + processing.callStack().getLastPopped(); if (exception != null) { if (exception instanceof HandledProcessingException) { errorMsg += ". Error message: " + exception.getMessage(); log.log(Level.WARNING, errorMsg); log.log(Level.FINE, "Chained exception:", exception); } else { log.log(Level.WARNING, errorMsg, exception); } } else { log.log(Level.WARNING, errorMsg); } //LOGGING OF STACK TRACE: if (exception != null) { StringWriter backtrace = new StringWriter(); exception.printStackTrace(new PrintWriter(backtrace)); log.log(LogLevel.DEBUG, "Failed to process " + processing + ": " + backtrace.toString()); } }
private Processing(DocumentOperation documentOperation) { this(); addDocumentOperation(documentOperation); }
/** * Convenience method for looking up and returning the service processing this. This might return null * if #getServiceName returns a name that is not registered in {@link com.yahoo.docproc.DocprocService}. * * @return the service processing this, or null if unknown. */ public DocprocService getService() { if (docprocServiceRegistry != null) { return docprocServiceRegistry.getComponent(getServiceName()); } return null; }
private void incrementNumDocsProcessed(Processing processing) { int increment = processing.getNumDocsToBeProcessed(); if (increment != 0) { incrementNumDocsProcessed(increment); } }
/** * Asynchronously process the given document operations as one unit * using the processing chain of this service, * and call the specified ProcessingEndpoint when done. * * @throws RuntimeException caused by a QueueFullException if this DocprocService has a bounded input queue and the queue is full * @throws IllegalStateException if this DocprocService is not accepting new incoming processings */ public void processDocumentOperations(List<DocumentOperation> documentOperations, ProcessingEndpoint endp) { addProcessing(Processing.createProcessingFromDocumentOperations(getName(), documentOperations, new CallStack(getCallStack()), endp)); }
/** * Processes the given Processing through the specified chain. Note that if one * {@link com.yahoo.docproc.DocumentProcessor DocumentProcessor} in the * chain returns a {@link com.yahoo.docproc.DocumentProcessor.LaterProgress DocumentProcessor.LaterProgress}, * the calling thread will sleep for the duration * specified in {@link com.yahoo.docproc.DocumentProcessor.LaterProgress#getDelay() DocumentProcessor.LaterProgress#getDelay()}, * and then run again. This method will hence return when a document processor returns * {@link com.yahoo.docproc.DocumentProcessor.Progress#DONE DocumentProcessor.Progress#DONE} or * {@link com.yahoo.docproc.DocumentProcessor.Progress#FAILED DocumentProcessor.Progress#FAILED}, throws an exception, * or if the calling thread is interrupted. This method will never return a * {@link com.yahoo.docproc.DocumentProcessor.LaterProgress DocumentProcessor.LaterProgress}. * * @param chain the specification of the chain to execute * @param processing the Processing to process * @return Progress.DONE or Progress.FAILED * @throws RuntimeException if one of the document processors in the chain throws, or if the calling thread is interrupted */ public DocumentProcessor.Progress process(ComponentSpecification chain, com.yahoo.docproc.Processing processing) { DocprocExecutor executor = getExecutor(chain); processing.setDocprocServiceRegistry(handler.getDocprocServiceRegistry()); return executor.processUntilDone(processing); }
/** * * @param service the unique name of the service processing this * @param documentsAndUpdates the document operation list. This <b>transfers ownership</b> of this list * to this class. The caller <i>must not</i> modify it * @param callStack the document processors to call in this processing. * This <b>transfers ownership</b> of this structure * to this class. The caller <i>must not</i> modify it */ public static Processing createProcessingFromDocumentOperations(String service, List<DocumentOperation> documentsAndUpdates, CallStack callStack) { return new Processing(service, documentsAndUpdates, callStack, null, false); }
final String location; if (processing != null) { final CallStack callStack = processing.callStack(); if (callStack != null) { final Call lastPopped = callStack.getLastPopped();
private void schemaMapProcessing(Processing processing) { final List<DocumentOperation> documentOperations = processing.getDocumentOperations(); for (int i = 0; i < documentOperations.size(); i++) { DocumentOperation op = documentOperations.get(i); if (op instanceof DocumentPut) { documentOperations.set(i, configDoc(processor, (DocumentPut) op)); } else if (op instanceof DocumentUpdate) { documentOperations.set(i, configDocUpd(processor, (DocumentUpdate) op)); } } }
static Processing createProcessingFromDocumentOperations(String service, List<DocumentOperation> documentOpsAndUpdates, CallStack callStack, ProcessingEndpoint endp) { return new Processing(service, documentOpsAndUpdates, callStack, endp, false); }
@Override public Progress process(Processing processing) { if (processing.getDocumentOperations().size() != 1) { //we were given more than one document, return log.log(LogLevel.DEBUG, "More than one document given, returning. (Was given " + processing.getDocumentOperations().size() + " documents)."); return Progress.DONE; } if (!doProcessOuterDocument(processing.getDocumentOperations().get(0), documentTypeName)) { return Progress.DONE; } Document outerDoc = ((DocumentPut)processing.getDocumentOperations().get(0)).getDocument();; @SuppressWarnings("unchecked") Array<Document> innerDocuments = (Array<Document>) outerDoc.getFieldValue(arrayFieldName); if (innerDocuments == null) { //the document does not have the field, return log.log(LogLevel.DEBUG, "The given Document does not have a field value for field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } if (innerDocuments.size() == 0) { //the array is empty, return log.log(LogLevel.DEBUG, "The given Document does not have any elements in array field " + arrayFieldName + ", returning. (Was given " + outerDoc + ")."); return Progress.DONE; } split(processing, innerDocuments); return Progress.DONE; }