public DataTierTupleSource(AtomicRequestMessage aqr, RequestWorkItem workItem, ConnectorWork cwi, DataTierManagerImpl dtm, int limit) { this.aqr = aqr; this.workItem = workItem; this.cwi = cwi; this.dtm = dtm; this.limit = limit; Assertion.isNull(workItem.getConnectorRequest(aqr.getAtomicRequestID())); workItem.addConnectorRequest(aqr.getAtomicRequestID(), this); }
public boolean blockOnOutputBuffer(RequestWorkItem item) { synchronized (waitingPlans) { if (!waitingPlans.isEmpty()) { return false; } if (item.useCallingThread || item.getDqpWorkContext().getSession().isEmbedded()) { return false; } } return true; }
/** * Special call from request threads to allow resumption of processing by * the calling thread. */ public void doMoreWork() { boolean run = false; synchronized (this) { moreWork(); if (!useCallingThread || this.getThreadState() != ThreadState.MORE_WORK) { return; } run = !hasThread; } if (run) { //run outside of the lock LogManager.logDetail(LogConstants.CTX_DQP, "Restarting processing using the calling thread", requestID); //$NON-NLS-1$ run(); } }
private void signalMore() { if (!done.get()) { synchronized (workItem) { if (workItem.getThreadState() != ThreadState.MORE_WORK) { workItem.moreWork(); } } } }
private void done() { doneProducingBatches(); addToCache(); //TODO: we could perform more tracking to know what source lobs are in use if (this.resultsBuffer.getLobCount() == 0) { for (DataTierTupleSource connectorRequest : getConnectorRequests()) { connectorRequest.fullyCloseSource(); } } }
RequestWorkItem holder = requests.get(requestID); if(holder != null && !holder.isCanceled() && (longRunningQueryThreshold == -1 || holder.getProcessingTimestamp() < longRunningQueryThreshold)) { RequestMetadata req = new RequestMetadata(); req.setSessionId(holder.requestID.getConnectionID()); req.setCommand(holder.requestMsg.getCommandString()); req.setStartTime(holder.getProcessingTimestamp()); req.setState(holder.isCanceled()?ProcessingState.CANCELED:(holder.isDoneProcessing() || holder.isCloseRequested())?ProcessingState.DONE:ProcessingState.PROCESSING); switch (holder.getThreadState()) { case DONE: case IDLE: break; default: if (holder.isProcessing()) { req.setThreadState(ThreadState.RUNNING); } else { if (holder.getTransactionContext() != null && holder.getTransactionContext().getTransactionType() != Scope.NONE) { req.setTransactionId(holder.getTransactionContext().getTransactionId()); for (DataTierTupleSource conInfo : holder.getConnectorRequests()) { String connectorName = conInfo.getConnectorName();
public void testRemoveConnectorInfo() { DQPCore rm = new DQPCore(); rm.setTransactionService(new FakeTransactionService()); RequestMessage r0 = new RequestMessage("foo"); //$NON-NLS-1$ RequestID requestID = new RequestID(SESSION_STRING, 1); RequestWorkItem workItem = addRequest(rm, r0, requestID, null, null); AtomicRequestMessage atomicReq = new AtomicRequestMessage(workItem.requestMsg, workItem.getDqpWorkContext(), 1); DataTierTupleSource info = Mockito.mock(DataTierTupleSource.class); workItem.addConnectorRequest(atomicReq.getAtomicRequestID(), info); workItem.closeAtomicRequest(atomicReq.getAtomicRequestID()); DataTierTupleSource arInfo = workItem.getConnectorRequest(atomicReq.getAtomicRequestID()); assertNull(arInfo); } }
LogManager.putMdc(REQUEST_KEY, requestID.toString()); try { while (!isDoneProcessing()) { super.run(); if (!useCallingThread) { break; //allow results to be processed by calling thread if (this.getThreadState() == ThreadState.MORE_WORK) { continue; } catch (InterruptedException e) { try { requestCancel(); } catch (TeiidComponentException e1) { throw new TeiidRuntimeException(QueryPlugin.Event.TEIID30543, e1);
public void testGetConnectorInfo() { DQPCore rm = new DQPCore(); rm.setTransactionService(new FakeTransactionService()); RequestMessage r0 = new RequestMessage("foo"); //$NON-NLS-1$ RequestID requestID = new RequestID(SESSION_STRING, 1); RequestWorkItem workItem = addRequest(rm, r0, requestID, null, null); AtomicRequestMessage atomicReq = new AtomicRequestMessage(workItem.requestMsg, workItem.getDqpWorkContext(), 1); DataTierTupleSource info = Mockito.mock(DataTierTupleSource.class); workItem.addConnectorRequest(atomicReq.getAtomicRequestID(), info); DataTierTupleSource arInfo = workItem.getConnectorRequest(atomicReq.getAtomicRequestID()); assertTrue(arInfo == info); }
if (!connectorInfo.isEmpty()) { if (explicitSourceClose) { for (DataTierTupleSource ts : getConnectorRequests()) { if (!ts.isExplicitClose()) { timeOut = 100; moreWorkTask = null; if (getThreadState() != ThreadState.MORE_WORK) { moreWorkTask = scheduleWork(timeOut);
final RequestWorkItem workItem = new RequestWorkItem(this, requestMsg, request, resultsFuture.getResultsReceiver(), requestID, workContext); logMMCommand(workItem, Event.NEW, null, null); addRequest(requestID, workItem, state); workItem.setCancelTask(this.cancellationTimer.add(new Runnable() { WeakReference<RequestWorkItem> workItemRef = new WeakReference<RequestWorkItem>(workItem); @Override workItem.run();
private void sendError() { ResultsReceiver<ResultsMessage> receiver = null; synchronized (this) { receiver = this.resultsReceiver; this.resultsReceiver = null; if (receiver == null) { LogManager.logDetail(LogConstants.CTX_DQP, processingException, "Unable to send error to client as results were already sent.", requestID); //$NON-NLS-1$ return; } } LogManager.logDetail(LogConstants.CTX_DQP, processingException, "Sending error to client", requestID); //$NON-NLS-1$ ResultsMessage response = new ResultsMessage(); Throwable exception = this.processingException; if (this.options.isSanitizeMessages() && !LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { //we still convey an exception hierarchy here because the client logic looks for certian exception types exception = ExceptionUtil.sanitize(exception, false); } if (isCanceled) { exception = addCancelCode(exception); } setWarnings(response); response.setException(exception); setAnalysisRecords(response); receiver.receiveResults(response); }
@Override public void release() { try { requestCancel(); } catch (TeiidComponentException e) { LogManager.logWarning(LogConstants.CTX_DQP, e, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30026,requestID)); } }
public void requestClose() throws TeiidComponentException { if (this.state == ProcessingState.CLOSE || this.closeRequested) { if (LogManager.isMessageToBeRecorded(LogConstants.CTX_DQP, MessageLevel.DETAIL)) { LogManager.logDetail(LogConstants.CTX_DQP, "Request already closing" + requestID); //$NON-NLS-1$ } return; } if (!this.doneProducingBatches) { this.requestCancel(); //pending work should be canceled for fastest clean up } else { //it's safe to transition directly to close this.state = ProcessingState.CLOSE; } this.closeRequested = true; this.doMoreWork(); }
@Override public void onCompletion(FutureWork<T> future) { WorkWrapper<?> nextWork = null; synchronized (queue) { if (!submitted) { return; } synchronized (RequestWorkItem.this) { if (isProcessing()) { totalThreads--; moreWork(); return; } } nextWork = queue.pollFirst(); if (nextWork == null) { totalThreads--; } else { nextWork.submitted = true; } } if (nextWork != null) { dqpCore.addWork(nextWork.work); } } }
private AtomicRequestMessage createRequest(RequestWorkItem workItem, Command command, String modelName, String connectorBindingId, int nodeID) throws TeiidComponentException { RequestMessage request = workItem.requestMsg; // build the atomic request based on original request + context info AtomicRequestMessage aqr = new AtomicRequestMessage(request, workItem.getDqpWorkContext(), nodeID); aqr.setCommand(command); aqr.setModelName(modelName); aqr.setMaxResultRows(requestMgr.getMaxSourceRows()); aqr.setExceptionOnMaxRows(requestMgr.isExceptionOnMaxSourceRows()); aqr.setPartialResults(request.supportsPartialResults()); aqr.setSerial(requestMgr.getUserRequestSourceConcurrency() == 1); aqr.setTransactionContext(workItem.getTransactionContext()); aqr.setBufferManager(this.getBufferManager()); if (connectorBindingId == null) { VDBMetaData vdb = workItem.getDqpWorkContext().getVDB(); ModelMetaData model = vdb.getModel(modelName); List<String> bindings = model.getSourceNames(); if (bindings == null || bindings.size() != 1) { // this should not happen, but it did occur when setting up the SystemAdmin models throw new TeiidComponentException(QueryPlugin.Event.TEIID30554, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID30554, modelName, workItem.getDqpWorkContext().getVdbName(), workItem.getDqpWorkContext().getVdbVersion())); } connectorBindingId = bindings.get(0); Assertion.isNotNull(connectorBindingId, "could not obtain connector id"); //$NON-NLS-1$ } aqr.setConnectorName(connectorBindingId); return aqr; }
Assertion.isNotNull(workItem); if(CoreConstants.SYSTEM_MODEL.equals(modelName) || CoreConstants.SYSTEM_ADMIN_MODEL.equals(modelName)) { return processSystemQuery(context, command, workItem.getDqpWorkContext()); ConnectorManagerRepository cmr = workItem.getDqpWorkContext().getVDB().getAttachment(ConnectorManagerRepository.class); ConnectorManager connectorManager = cmr.getConnectorManager(aqr.getConnectorName()); if (connectorManager == null) { if (workItem.getRsCache() != null && command.areResultsCachable()) { CachableVisitor cv = new CachableVisitor(); PreOrPostOrderNavigator.doVisit(command, cv, PreOrPostOrderNavigator.PRE_ORDER, true); String cmdString = command.toString(); if (cmdString.length() < 100000) { //TODO: this check won't be needed if keys aren't exclusively held in memory cid = new CacheID(workItem.getDqpWorkContext(), ParseInfo.DEFAULT_INSTANCE, cmdString); cid.setParameters(cv.parameters); if (cd.getInvalidation() == null || cd.getInvalidation() == Invalidation.NONE) { CachedResults cr = workItem.getRsCache().get(cid); if (cr != null && (cr.getRowLimit() == 0 || (parameterObject.limit > 0 && cr.getRowLimit() >= parameterObject.limit))) { parameterObject.doNotCache = true; workItem.getRsCache().remove(cid, CachingTupleSource.getDeterminismLevel(cd.getScope()));
@Test public void testDataTierTupleSourceWarnings() throws Exception { DataTierTupleSource info = helpSetup(1); connectorManager.addWarning = true; assertEquals(10, pullTuples(info, 10)); assertNotNull(workItem.getConnectorRequest(info.getAtomicRequestMessage().getAtomicRequestID())); assertNull(info.nextTuple()); List<Exception> warnings = context.getAndClearWarnings(); assertEquals(1, warnings.size()); SourceWarning warning = (SourceWarning) warnings.get(0); assertFalse(warning.isPartialResultsError()); info.closeSource(); assertNull(workItem.getConnectorRequest(info.getAtomicRequestMessage().getAtomicRequestID())); }
public void fullyCloseSource() { cancelFutures(); cancelAsynch = true; if (closing.compareAndSet(false, true)) { if (!done && !errored) { this.cwi.cancel(false); } workItem.closeAtomicRequest(this.aqr.getAtomicRequestID()); if (aqr.isSerial() || futureResult == null) { this.cwi.close(); } else { futureResult.addCompletionListener(new CompletionListener<AtomicResultsMessage>() { @Override public void onCompletion(FutureWork<AtomicResultsMessage> future) { if (running) { return; //-- let the other thread close } if (closed.compareAndSet(false, true)) { cwi.close(); } } }); } } }
static RequestWorkItem addRequest(DQPCore rm, RequestMessage requestMsg, RequestID id, Command originalCommand, DQPWorkContext workContext) { if (workContext == null) { workContext = new DQPWorkContext(); workContext.getSession().setSessionId(id.getConnectionID()); workContext.getSession().setUserName("foo"); //$NON-NLS-1$ } RequestWorkItem workItem = new RequestWorkItem(rm, requestMsg, null, null, id, workContext); workItem.setOriginalCommand(originalCommand); ClientState state = rm.getClientState(id.getConnectionID(), true); rm.addRequest(id, workItem, state); return workItem; }