private synchronized QueuedWork getIncompleteWork( String container, String testrig, WorkType wType) { for (QueuedWork work : _queueIncompleteWork) { WorkDetails wDetails = work.getDetails(); if (container.equals(work.getWorkItem().getContainerName()) && ((testrig.equals(wDetails.baseTestrig)) || (wDetails.isDifferential && testrig.equals(wDetails.deltaTestrig))) && (wType == null || wDetails.workType == wType)) { return work; } } return null; }
public synchronized List<QueuedWork> listIncompleteWork( String containerName, @Nullable String testrigName, @Nullable WorkType workType) { List<QueuedWork> retList = new LinkedList<>(); for (QueuedWork work : _queueIncompleteWork) { // Add to queue if it matches container, testrig if provided, and work type if provided if (work.getWorkItem().getContainerName().equals(containerName) && (testrigName == null || work.getDetails().baseTestrig.equals(testrigName)) && (workType == null || work.getDetails().workType == workType)) { retList.add(work); } } return retList; }
/** * Get all completed work for the specified network and snapshot. * * @param network {@link NetworkId} to get completed work for. * @param snapshot {@link SnapshotId} to get completed work for. * @return {@link List} of completed {@link QueuedWork}. */ public synchronized List<QueuedWork> getCompletedWork(NetworkId network, SnapshotId snapshot) { ImmutableList.Builder<QueuedWork> b = ImmutableList.builder(); for (QueuedWork work : _queueCompletedWork) { if (work.getWorkItem().getContainerName().equals(network.getId()) && work.getWorkItem().getTestrigName().equals(snapshot.getId())) { b.add(work); } } return b.build(); }
public synchronized void markAssignmentSuccess(QueuedWork work, String assignedWorker) throws IOException { work.setAssignment(assignedWorker); // update testrig metadata WorkItem wItem = work.getWorkItem(); WorkDetails wDetails = work.getDetails(); if (wDetails.workType == WorkType.PARSING) { WorkQueueMgr.updateInitializationStatus( wItem.getContainerName(), wDetails.baseTestrig, ProcessingStatus.PARSING, null); } else if (wDetails.workType == WorkType.DATAPLANING) { WorkQueueMgr.updateInitializationStatus( wItem.getContainerName(), wDetails.baseTestrig, ProcessingStatus.DATAPLANING, null); } }
WorkQueueMgr.getInitializationMetadata(wItem.getContainerName(), testrig); QueuedWork parsingWork = getIncompleteWork(wItem.getContainerName(), testrig, WorkType.PARSING); getIncompleteWork(wItem.getContainerName(), testrig, WorkType.DATAPLANING); return dataplaningWork; return generateAndQueueDataplaneWork(wItem.getContainerName(), testrig); case DATAPLANING_FAIL: case DATAPLANING:
private QueuedWork getBlockerForParsingDependentWork(QueuedWork work, String testrig) throws IOException { WorkItem wItem = work.getWorkItem(); InitializationMetadata metadata = WorkQueueMgr.getInitializationMetadata(wItem.getContainerName(), testrig); QueuedWork parsingWork = getIncompleteWork(wItem.getContainerName(), testrig, WorkType.PARSING); switch (metadata.getProcessingStatus()) { case UNINITIALIZED: case PARSING: case PARSING_FAIL: if (parsingWork == null) { throw new BatfishException( String.format( "Cannot queue parsing dependent work for %s: " + "Status is %s but no incomplete parsing work exists", testrig, metadata.getProcessingStatus())); } return parsingWork; case PARSED: case DATAPLANING: case DATAPLANED: case DATAPLANING_FAIL: return parsingWork; default: throw new BatfishException( "Unknown snapshot processingStatus: " + metadata.getProcessingStatus()); } }
private synchronized boolean queueParsingWork(QueuedWork work) throws Exception { WorkItem wItem = work.getWorkItem(); WorkDetails wDetails = work.getDetails(); // if incomplete work for this testrig exists, lets just reject this parsing work. // parsing work cannot proceeed in parallel because it may overwrite files used by others. // instead of rejecting, we could have queued it as BLOCKED but we risk cycles of BLOCKED work // this should not be a common case anyway, so we aren't losing much by rejecting it QueuedWork incompleteWork = getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, null); if (incompleteWork != null) { throw new BatfishException("Cannot queue parsing work while other work is incomplete"); } else { InitializationMetadata metadata = WorkQueueMgr.getInitializationMetadata(wItem.getContainerName(), wDetails.baseTestrig); if (metadata.getProcessingStatus() == ProcessingStatus.PARSING) { throw new BatfishException( String.format( "Cannot queue parsing work for %s: " + "Status is PARSING but no incomplete parsing work exists", wDetails.baseTestrig)); } else if (metadata.getProcessingStatus() == ProcessingStatus.DATAPLANING) { throw new BatfishException( String.format( "Cannot queue parsing work for %s: " + "Status is DATAPLANING but no incomplete dataplaning work exists", wDetails.baseTestrig)); } } return _queueIncompleteWork.enque(work); }
private boolean queueDataplaningWork(QueuedWork work) throws Exception { WorkItem wItem = work.getWorkItem(); WorkDetails wDetails = work.getDetails(); QueuedWork currentDataplaningWork = getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, WorkType.DATAPLANING); if (currentDataplaningWork != null) { throw new BatfishException("Dataplaning is already in queue/progress"); } // see comment in queueParsingWork for justification QueuedWork ddWork = getIncompleteWork( wItem.getContainerName(), wDetails.baseTestrig, WorkType.DATAPLANE_DEPENDENT_ANSWERING); if (ddWork != null) { throw new BatfishException("Cannot queue dataplaning work while other dependent work exists"); } QueuedWork blocker = getBlockerForDataplaningWork(work); if (blocker == null) { return _queueIncompleteWork.enque(work); } else { return queueBlockedWork(work, blocker); } }
public boolean queueWork(WorkItem workItem) { NetworkId networkId = _idManager.getNetworkId(requireNonNull(workItem.getContainerName())); boolean success; try {
String network = workItem.getContainerName(); if (network == null) { return workItem;
getIncompleteWork(wItem.getContainerName(), wDetails.baseTestrig, WorkType.PARSING); WorkQueueMgr.getInitializationMetadata(wItem.getContainerName(), wDetails.baseTestrig);
String networkId = work.getWorkItem().getContainerName(); Optional<String> networkOpt = Main.getWorkMgr().getNetworkNames().stream()
_workHelper.getObject(wItem.getContainerName(), wItem.getTestrigName(), ansFileName); if (downloadedAnsFile == null) { _logger.errorf("Failed to get answer file %s. (Was work killed?)\n", ansFileName); String logFileName = wItem.getId() + BfConsts.SUFFIX_LOG_FILE; String downloadedFileStr = _workHelper.getObject(wItem.getContainerName(), wItem.getTestrigName(), logFileName);
checkNetworkAccessibility(apiKey, work.getWorkItem().getContainerName());
checkNetworkAccessibility(apiKey, workItem.getContainerName());
if (!workItem.getContainerName().equals(networkName) || !workItem.getTestrigName().equals(snapshotName)) { return failureResponse(
private QueuedWork resolvedQueuedWork(WorkItem workItem, WorkDetails workDetails) { WorkItem resolvedWorkItem = WorkMgr.resolveIds(workItem); return new QueuedWork( WorkMgr.resolveIds(workItem), resolveIds(new NetworkId(resolvedWorkItem.getContainerName()), workDetails)); }
Question.parseQuestion( getQuestion( workItem.getContainerName(), WorkItemBuilder.getQuestionName(workItem), WorkItemBuilder.getAnalysisName(workItem))); throw new BatfishException("Analysis name not provided for ANALYZE work"); Set<String> qNames = listAnalysisQuestions(workItem.getContainerName(), aName); Question.parseQuestion(getQuestion(workItem.getContainerName(), qName, aName)); if (question.getDataPlane()) { workType = WorkType.DATAPLANE_DEPENDENT_ANSWERING;
public synchronized boolean queueUnassignedWork(QueuedWork work) throws Exception { QueuedWork previouslyQueuedWork = getWork(work.getId()); if (previouslyQueuedWork != null) { throw new BatfishException("Duplicate work item"); } WorkDetails wDetails = work.getDetails(); cleanUpInitMetaDataIfNeeded(work.getWorkItem().getContainerName(), wDetails.baseTestrig); if (work.getDetails().isDifferential) { cleanUpInitMetaDataIfNeeded(work.getWorkItem().getContainerName(), wDetails.deltaTestrig); } switch (work.getDetails().workType) { case PARSING: return queueParsingWork(work); case DATAPLANING: return queueDataplaningWork(work); case INDEPENDENT_ANSWERING: // assume that this type of work shouldn't be blocked at all return _queueIncompleteWork.enque(work); case PARSING_DEPENDENT_ANSWERING: return queueDependentAnsweringWork(work, false); case DATAPLANE_DEPENDENT_ANSWERING: return queueDependentAnsweringWork(work, true); case UNKNOWN: return _queueIncompleteWork.enque(work); default: throw new BatfishException("Unknown WorkType " + work.getDetails().workType); } }
if (!workItem.getContainerName().equals(networkName) || !workItem.getTestrigName().equals(snapshotName)) { return failureResponse(