/** * Composes an {@link Package.Events#INGEST_FAIL INGEST_FAIL} event, setting the event outcome to the first line of * the supplied {@code Exception}, and putting the full stack trace of the {@code Exception} in the event detail. * The target of the event is set to the {@code depositId}. * <p/> * This method does <em>not</em> add the event to the ingest state. * * @param depositId the id of the deposit that failed * @param state the ingest state associated with the deposit, used to create the failure event * @param t the exception that caused the deposit to fail * @return the failure event, with the date, outcome, and detail populated */ private DcsEvent failureEvent(String depositId, IngestWorkflowState state, Throwable t) { final DcsEvent failureEvent = state.getEventManager().newEvent(Package.Events.INGEST_FAIL); failureEvent.setOutcome("Ingest " + depositId + " failed: " + t.getMessage()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(baos); t.printStackTrace(ps); failureEvent.setDetail(new String(baos.toByteArray())); failureEvent.addTargets(new DcsEntityReference(depositId)); return failureEvent; }
/** * Creates a {@link org.dataconservancy.packaging.ingest.api.Package.Events#FILE_EXTRACTION} event, representing all of the files that were extracted from * the Package, and adds it to the ingest state. * * @param depositId the deposit (transaction) identifier * @param archiveFileName the original name of the package file that was uploaded * @param extractedFiles a list of files extracted from the package * @param state the ingest state * @throws IOException */ void addExtractionEvent(String depositId, String archiveFileName, List<File> extractedFiles, IngestWorkflowState state) throws IOException { final PackageSerialization ser = state.getPackage().getSerialization(); DcsEvent extractionEvent = state.getEventManager().newEvent(Package.Events.FILE_EXTRACTION); extractionEvent.setOutcome(String.valueOf(extractedFiles.size())); extractionEvent.setDetail("Extracted " + extractedFiles.size() + " files from " + archiveFileName + " to " + new File(ser.getExtractDir(), ser.getBaseDir().getPath()).getAbsolutePath() + "(" + depositId + ")"); for (File f : extractedFiles) { extractionEvent.addTargets(new DcsEntityReference(f.getCanonicalPath())); } state.getEventManager().addEvent(depositId, extractionEvent); }
@Override public void execute(String depositId, IngestWorkflowState state) throws StatefulIngestServiceException { super.execute(depositId, state); List<DcsEntityReference> businessObjectRefs = build(state, depositId); // Emit event indication that business objects had been created from attribute sets DcsEvent event = state.getEventManager().newEvent(Package.Events.BUSINESS_OBJECT_BUILT); event.setDetail(SUCCESS_EVENT_DETAIL); event.setOutcome(SUCCESS_EVENT_OUTCOME + businessObjectRefs.size() + " objects created."); event.setTargets(businessObjectRefs); state.getEventManager().addEvent(depositId, event); }
/** * Composes an {@link Package.Events#INGEST_SUCCESS} event and adds it to the workflow state. The event is only * added if the last phase has been completed, otherwise this method simply returns. * * @param start the start time in milliseconds of the ingest * @param state the ingest state associated with the deposit ID * @param depositId the deposit DI of the successful ingest */ private void addSuccessEvent(long start, IngestWorkflowState state, String depositId) { DcsEvent event = state.getEventManager().newEvent(Package.Events.INGEST_SUCCESS); event.setDetail("Ingest of deposit: " + depositId + " successful"); event.setOutcome(depositId); // TODO DC-1477: The target should probably enumerate the business ids of the objects that were added to the system. List<DcsEntityReference> refs = new ArrayList<DcsEntityReference>(); DcsEntityReference ref = new DcsEntityReference(depositId); refs.add(ref); event.setTargets(refs); state.getEventManager().addEvent(depositId, event); }
@Override public void execute(String depositId, IngestWorkflowState state) throws StatefulIngestServiceException { super.execute(depositId, state); BusinessObjectManager businessObjectManager = state.getBusinessObjectManager(); List<DcsEntityReference> references = checkBusinessObjectsValidity(businessObjectManager); if (references.size() > 0) { // Emit event that indicates built business objects are valid. DcsEvent event = state.getEventManager().newEvent(Package.Events.BUSINESS_OBJECTS_VALIDATED); event.setDetail("Successfully validated the built business objects."); event.setOutcome("Successfully validated a total number of: " + references.size() + " BusinessObjects."); event.setTargets(references); state.getEventManager().addEvent(depositId, event); } }
@Override public void execute(String depositId, IngestWorkflowState state) throws StatefulIngestServiceException { super.execute(depositId, state); Set<DcsEntityReference> metadataFilesCreated = buildAttributeSetMetadataFile(state); //Emit event indication that business objects had been created from attribute sets DcsEvent event = state.getEventManager().newEvent(org.dataconservancy.packaging.ingest.api.Package.Events.BUSINESS_OBJECT_BUILT); event.setDetail(SUCCESS_EVENT_DETAIL); event.setOutcome(SUCCESS_EVENT_OUTCOME + metadataFilesCreated.size() + " MetadataFile objects created."); event.setTargets(metadataFilesCreated); state.getEventManager().addEvent(depositId, event); }
private void sendEvent(EventManager manager, String id, String outcome, String details, MetadataFile file) { // Send the event, using the outcome and any error/warning messages. DcsEvent event = manager.newEvent(Package.Events.METADATA_FILE_VALIDATED); event.setOutcome(outcome); event.setDetail(details); event.addTargets(new DcsEntityReference(file.getId())); manager.addEvent(id, event); } }
private void sendEvent(EventManager manager, String id, String outcome, String details, MetadataFile file) { // Send the event, using the outcome and any error/warning messages. DcsEvent event = manager.newEvent(Package.Events.METADATA_FILE_VALIDATED); event.setOutcome(outcome); event.setDetail(details); event.addTargets(new DcsEntityReference(file.getId())); manager.addEvent(id, event); } }
/** * Composes and adds a phase complete event to the state. * * @param depositId the deposit id * @param state the state associated with the deposit */ private void addPhaseComplete(long start, String depositId, IngestWorkflowState state, List<Class<? extends StatefulIngestService>> completedServices) { DcsEvent e = state.getEventManager().newEvent(Package.Events.INGEST_PHASE_COMPLETE); // outcome e.setOutcome(String.valueOf(state.getIngestPhase().getPhaseNumber())); // detail e.setDetail("Completed ingest phase " + state.getIngestPhase().getPhaseNumber() + " for deposit id " + depositId); // target for (Class<? extends StatefulIngestService> service : completedServices) { e.addTargets(new DcsEntityReference(service.getName())); } state.getEventManager().addEvent(depositId, e); log.info("Ingest phase {} (of {}) for deposit id {} completed in {} ms", new Object[]{state.getIngestPhase().getPhaseNumber(), ingestServicesMap.keySet().size(), depositId, (System.currentTimeMillis() - start)}); }
private boolean findAllAttributes(String depositId, IngestWorkflowState state, String attributeName, Set<String> ids) { Attribute attributeToMatch = new AttributeImpl(attributeName, null, null); Set<AttributeSet> matchingAttributeSets = state.getAttributeSetManager().matches(attributeToMatch); for (AttributeSet attributeSet : matchingAttributeSets) { for (Attribute matchingAttribute : attributeSet.getAttributes()) { if (matchingAttribute.getName().equalsIgnoreCase(attributeName)) { if (!ids.add(matchingAttribute.getValue())) { DcsEvent event = state.getEventManager().newEvent(Package.Events.INGEST_FAIL); String stackTrace = ""; for (StackTraceElement st : Thread.currentThread().getStackTrace()) { stackTrace += st + "\n"; } event.setDetail(stackTrace); event.setOutcome("Duplicate Identifiers were found: " + matchingAttribute.getValue()); List<DcsEntityReference> refs = new ArrayList<DcsEntityReference>(); DcsEntityReference ref = new DcsEntityReference(matchingAttribute.getValue()); refs.add(ref); event.setTargets(refs); state.getEventManager().addEvent(depositId, event); return false; } } } } return true; } }
/** * Creates a {@link org.dataconservancy.packaging.ingest.api.Package.Events#DEPOSIT} event, representing the receipt of the Package, and adds it to the * ingest state. * * @param metadata map of metadata using HTTP header conventions * @param depositId the deposit (transaction) identifier * @param state the ingest state */ void addDepositEvent(Map<String, String> metadata, String depositId, IngestWorkflowState state) { DcsEvent depositEvent = state.getEventManager().newEvent(org.dataconservancy.packaging.ingest.api.Package.Events.DEPOSIT); depositEvent.setOutcome(depositId); depositEvent.addTargets(new DcsEntityReference(filenameFromHeader(metadata))); depositEvent.setDetail("Accepted package named " + filenameFromHeader(metadata) + " on " + depositEvent.getDate() + " deposited by " + metadata.get(Http.Header.X_DCS_AUTHENTICATED_USER)); state.getEventManager().addEvent(depositId, depositEvent); }
@Test public void creatAndPersistTest() { EventManager mgr = getEventManager(); DcsEvent e = mgr.newEvent(TEST_EVENT_TYPE); e.setDetail("detail"); e.setOutcome("outcome"); e.setTargets(getTargets()); String id = getSipStager().addSIP(new Dcp()); mgr.addEvent(id, e); Collection<DcsEvent> retrieved = mgr.getEvents(id); assertEquals("Expected to see one event", 1, retrieved.size()); assertTrue("Did not see test event", retrieved.contains(e)); }
@Override public void execute(String depositId, IngestWorkflowState state) throws StatefulIngestServiceException { super.execute(depositId, state); PackageSerialization pkg = state.getPackage().getSerialization(); State extractionState = new State(); // Get absolute base directory containing the package as a directory. extractionState.baseDir = new File(pkg.getExtractDir(), pkg.getBaseDir().getPath()).getParentFile(); extractionState.manager = state.getAttributeSetManager(); extractionState.model = ModelFactory.createDefaultModel(); AttributeSetManager manager = state.getAttributeSetManager(); URI uri = ResourceMapLoader.getPackageResourceMapURI(manager); Resource resmap = extractionState.model.createResource(uri.toString()); // Walk the graph constructing attribute sets for each object and // generate events. visitResourceMap(resmap, extractionState); DcsEvent event = state.getEventManager().newEvent( org.dataconservancy.packaging.ingest.api.Package.Events.TRANSFORM); event.setDetail("AttributeSets generated for the ORE-ReM with deposit ID: " + depositId); event.setOutcome("Succeeded in extracting Attributes from ORE-ReM"); DcsEntityReference ref = new DcsEntityReference(depositId); List<DcsEntityReference> refs = new ArrayList<DcsEntityReference>(); refs.add(ref); event.setTargets(refs); state.getEventManager().addEvent(depositId, event); }
/** * Composes an {@link Package.Events#INGEST_START} event and adds it to the workflow state. * * @param depositId the deposit ID of the ingest being started * @param state the ingest state associated with the deposit ID */ private void addStartEvent(String depositId, IngestWorkflowState state) { DcsEvent ingestStart = state.getEventManager().newEvent(Package.Events.INGEST_START); ingestStart.setOutcome(depositId); ingestStart.setDetail("Ingest " + depositId + " started"); for (File file : state.getPackage().getSerialization().getFiles()) { ingestStart.addTargets(new DcsEntityReference(file.getPath())); } state.getEventManager().addEvent(depositId, ingestStart); }
/** * Composes and adds a phase start event to the state. * * @param depositId the deposit id * @param state the state associated with the deposit */ private void addPhaseStartEvent(String depositId, IngestWorkflowState state) { DcsEvent e = state.getEventManager().newEvent(Package.Events.INGEST_PHASE_START); // outcome e.setOutcome(String.valueOf(state.getIngestPhase().getPhaseNumber())); // detail e.setDetail("Started ingest phase " + state.getIngestPhase().getPhaseNumber() + " for deposit id " + depositId); // target for (StatefulIngestService service : ingestServicesMap.get(state.getIngestPhase())) { e.addTargets(new DcsEntityReference(service.getName())); } state.getEventManager().addEvent(depositId, e); }
public DcsEvent createEvent(String target) { DcsEvent event = new DcsEvent(); event.setId(nextid()); event.setDate(DateUtility.toIso8601(DateUtility.now())); event.setOutcome(randomString(20, false)); event.setEventType(randomString(2, false)); event.setDetail(randomText(5)); event.setTargets(createEnityReferences(rand.nextInt(2))); return event; }
@Override public void execute(String depositId, IngestWorkflowState state) throws StatefulIngestServiceException { for (DcsEvent event : state.getEventManager().getEvents(depositId, Package.Events.METADATA_FILE_VALIDATED)) { if (!event.getTargets().isEmpty()) { if (event.getOutcome().equals(Package.Outcomes.SUCCESS)) { MetadataFile file = (MetadataFile)state.getBusinessObjectManager().get(event.getTargets().iterator().next().getRef()); MetadataResult result = metadataBizService.extractMetadata(file); String outcome = Package.Outcomes.SUCCESS; String extractionMessages = ""; if (!result.getMetadataExtractionErrors().isEmpty()) { outcome = Package.Outcomes.FAILURE; for (MetadataEventMessage error : result.getMetadataExtractionErrors()) { extractionMessages += error.getMessage() + "<\br>"; } } // Send the event, using the outcome and any error/warning messages. DcsEvent extractionEvent = state.getEventManager().newEvent(Package.Events.METADATA_FILE_EXTRACTION); extractionEvent.setOutcome(outcome); extractionEvent.setDetail(extractionMessages); extractionEvent.addTargets(new DcsEntityReference(file.getId())); state.getEventManager().addEvent(depositId, extractionEvent); } } } } }
@Override public void execute(String depositId, IngestWorkflowState state) throws StatefulIngestServiceException { for (DcsEvent event : state.getEventManager().getEvents(depositId, Package.Events.METADATA_FILE_VALIDATED)) { if (!event.getTargets().isEmpty()) { if (event.getOutcome().equals(Package.Outcomes.SUCCESS)) { MetadataFile file = (MetadataFile)state.getBusinessObjectManager().get(event.getTargets().iterator().next().getRef()); MetadataResult result = metadataBizService.extractMetadata(file); String outcome = Package.Outcomes.SUCCESS; String extractionMessages = ""; if (!result.getMetadataExtractionErrors().isEmpty()) { outcome = Package.Outcomes.FAILURE; for (MetadataEventMessage error : result.getMetadataExtractionErrors()) { extractionMessages += error.getMessage() + "<\br>"; } } // Send the event, using the outcome and any error/warning messages. DcsEvent extractionEvent = state.getEventManager().newEvent(Package.Events.METADATA_FILE_EXTRACTION); extractionEvent.setOutcome(outcome); extractionEvent.setDetail(extractionMessages); extractionEvent.addTargets(new DcsEntityReference(file.getId())); state.getEventManager().addEvent(depositId, extractionEvent); } } } } }
private static DcsEvent getEvent(SolrDocument doc) { DcsEvent event = new DcsEvent(); event.setId(getFirst(doc, EntityField.ID)); Date date = (Date) doc.getFirstValue(EventField.DATE.solrName()); if (date != null) { event.setDate(DateUtil.getThreadLocalDateFormat().format(date)); } event.setDetail(getFirst(doc, EventField.DETAIL)); event.setOutcome(getFirst(doc, EventField.OUTCOME)); event.setEventType(getFirst(doc, EventField.TYPE)); event.setTargets(getEntityRefSet(doc, EventField.TARGET)); event.setAlternateIds(getResourceIdentifierSet(doc)); return event; }
private String addIngestEvent(Dcp sip, String ingestId) { DcsEvent event = eventManager.newEvent(Events.DEPOSIT); event.setDetail(ingestId); for (DcsCollection c : sip.getCollections()) { event.addTargets(new DcsEntityReference(c.getId())); } for (DcsEntity du : sip.getDeliverableUnits()) { event.addTargets(new DcsEntityReference(du.getId())); } for (DcsManifestation m : sip.getManifestations()) { event.addTargets(new DcsEntityReference(m.getId())); } for (DcsFile f : sip.getFiles()) { event.addTargets(new DcsEntityReference(f.getId())); } for (DcsEvent e : sip.getEvents()) { event.addTargets(new DcsEntityReference(e.getId())); } eventManager.addEvent(ingestId, event); return ingestId; }