/** * Invoke when the file is completed processing */ public void complete() { final long failed = entitiesFailed.get() + connectionsFailed.get(); final FileImport.State state; final String message; if ( failed > 0 ) { state = FileImport.State.FAILED; message = fileImport.getErrorMessage() == null ? ERROR_MESSAGE : fileImport.getErrorMessage(); } else { state = FileImport.State.FINISHED; message = null; } updateFileImport( state, message ); }
/** * Create an instance to track counters. Note that when this instance is created, it will * attempt to load it's state from the entity manager. In the case of using this when resuming, * be sure you begin processing where the system thinks * it has left off. * * @param emf Entity Manager Factory * @param fileImport File Import Entity * @param flushCount The number of success + failures to accumulate before flushing */ public FileImportTracker( final EntityManagerFactory emf, final FileImport fileImport, final int flushCount ) { this.emf = emf; this.flushCount = flushCount; this.fileImport = fileImport; this.entitiesWritten.addAndGet( fileImport.getImportedEntityCount() ); this.entitiesFailed.addAndGet( fileImport.getFailedEntityCount() ); this.connectionsWritten.addAndGet( fileImport.getImportedConnectionCount() ); this.connectionsFailed.addAndGet( fileImport.getFailedConnectionCount() ); }
@Override public void doWrite(EntityManager em, FileImport fileImport, FileImportTracker stats) { try { if (logger.isTraceEnabled()) { logger.trace("Adding map to {}:{} dictionary {}", ownerEntityRef.getType(), ownerEntityRef.getType(), dictionaryName); } em.addMapToDictionary(ownerEntityRef, dictionaryName, dictionary); } catch (Exception e) { logger.error("Error writing dictionary. From file: " + fileImport.getFileName(), e); // TODO add statistics for dictionary writes and failures } } }
public FileImport() { setLastUpdatedUUID(" "); setErrorMessage(" "); setState(FileImport.State.CREATED); }
fileImport.setImportedEntityCount( writtenEntities ); fileImport.setFailedEntityCount( failedEntities ); fileImport.setImportedConnectionCount( writtenConnections ); fileImport.setFailedConnectionCount( failedConnections ); fileImport.setState( state ); fileImport.setErrorMessage( message );
final FileImport fileImport = new FileImport(); fileImport.setUuid( importFileId ); assertEquals( "Same count expected", expectedSuccess, updated.getImportedEntityCount() ); assertEquals( "Same fail expected", expectedFails, updated.getFailedEntityCount() ); updated.getImportedConnectionCount() ); updated.getFailedConnectionCount() ); assertTrue(updated.getErrorMessage().startsWith("Failed to import") );
@Test public void explicitFail() throws Exception { final EntityManagerFactory emf = mock( EntityManagerFactory.class ); final EntityManager em = mock( EntityManager.class ); when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em ); final UUID importFileId = UUIDGenerator.newTimeUUID(); final FileImport fileImport = new FileImport(); fileImport.setUuid( importFileId ); when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport ); final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 ); final long expectedCount = 100; for ( long i = 0; i < expectedCount; i++ ) { fileImportTracker.entityWritten(); } fileImportTracker.fatal( "Something bad happened" ); ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class ); verify( em ).update( savedFileImport.capture() ); final FileImport updated = savedFileImport.getValue(); assertSame( "Same instance should be updated", fileImport, updated ); assertEquals( "Same count expected", expectedCount, updated.getImportedEntityCount() ); assertEquals( "Fail count is 0", 0, updated.getFailedEntityCount() ); assertEquals( "Correct expected message", "Something bad happened", updated.getErrorMessage() ); assertEquals( "Expected failed state", FileImport.State.FAILED, updated.getState() ); }
@Test public void loadingExistingState() throws Exception { final EntityManagerFactory emf = mock( EntityManagerFactory.class ); final EntityManager em = mock( EntityManager.class ); when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em ); final UUID importFileId = UUIDGenerator.newTimeUUID(); final FileImport fileImport = new FileImport(); fileImport.setUuid( importFileId ); fileImport.setImportedEntityCount( 1 ); fileImport.setFailedEntityCount( 2 ); fileImport.setImportedConnectionCount( 3 ); fileImport.setFailedConnectionCount( 4 ); when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport ); //mock up returning the FailedEntityImport instance after save is invoked. FileImportTracker statistics = new FileImportTracker( emf, fileImport, 100 ); assertEquals( 1, statistics.getEntitiesWritten() ); assertEquals( 2, statistics.getEntitiesFailed() ); assertEquals( 3, statistics.getTotalEntityCount() ); assertEquals( 3, statistics.getConnectionsWritten() ); assertEquals( 4, statistics.getConnectionsFailed() ); assertEquals( 7, statistics.getTotalConnectionCount() ); }
@Test public void testSuccess() throws Exception { final EntityManagerFactory emf = mock( EntityManagerFactory.class ); final EntityManager em = mock( EntityManager.class ); when( emf.getEntityManager( emf.getManagementAppId() ) ).thenReturn( em ); final UUID importFileId = UUIDGenerator.newTimeUUID(); final FileImport fileImport = new FileImport(); fileImport.setUuid( importFileId ); when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport ); final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 ); final long expectedCount = 100; for ( long i = 0; i < expectedCount; i++ ) { fileImportTracker.entityWritten(); } fileImportTracker.complete(); ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class ); verify( em ).update( savedFileImport.capture() ); final FileImport updated = savedFileImport.getValue(); assertSame( "Same instance should be updated", fileImport, updated ); assertEquals( "Same count expected", expectedCount, updated.getImportedEntityCount() ); assertNull( updated.getErrorMessage() ); }
final FileImport fileImport = new FileImport(); final UUID importFileId = UUIDGenerator.newTimeUUID(); fileImport.setUuid(importFileId); assertEquals( "Same count expected", expectedSuccess, updated.getImportedEntityCount() ); assertEquals( "Same fail expected", expectedFails, updated.getFailedEntityCount() );
FileImport fileImport = new FileImport(file, applicationId); fileImport = rootEM.create(fileImport); logger.trace("Created connection from {}:{} to {}:{}", importEntity.getType(), importEntity.getUuid(), fileImport.getType(), fileImport.getUuid() ); fileImport.setState(FileImport.State.CREATED); rootEM.update(fileImport); jobData.setProperty(FILE_IMPORT_ID, fileImport.getUuid()); jobData.addProperties(config); fileImport.setState(FileImport.State.SCHEDULED); rootEM.update(fileImport);
if (FileImport.State.FAILED.equals(fileImport.getState())) { if (logger.isTraceEnabled()) { logger.trace("\n\nFailed to completely write entities, skipping second phase. File: {}\n", fileImport.getFileName()); logger.trace("\n\nWrote entities. File: {}\n", fileImport.getFileName()); fileImport.getFileName()); if (FileImport.State.FAILED.equals(fileImport.getState())) { if (logger.isDebugEnabled()) { logger.debug("\n\nparseEntitiesAndConnectionsFromJson(): failed to completely write entities\n"); if (FileImport.State.FAILED.equals(fileImport.getState())) { if (logger.isDebugEnabled()) { logger.debug("\n\nFailed to completely write connections and dictionaries. File: {}\n", fileImport.getFileName()); logger.trace("\n\nWrote connections and dictionaries. File: {}\n", fileImport.getFileName());
/** * This method is called when the job is retried maximum times by the scheduler but still fails. * Thus the scheduler marks it as DEAD. */ @Override public void dead( final JobExecution execution ) throws Exception { // Get the root entity manager EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() ); // Mark the sub-job i.e. File Import Job as Failed FileImport fileImport = null;//importService.getFileImportEntity(execution); fileImport.setErrorMessage("The Job has been tried maximum times but still failed"); fileImport.setState(FileImport.State.FAILED); rootEm.update(fileImport); // If one file Job fails, mark the main import Job also as failed Results ImportJobResults = rootEm.getSourceEntities( fileImport, "includes", null, Level.ALL_PROPERTIES); List<Entity> importEntity = ImportJobResults.getEntities(); UUID importId = importEntity.get(0).getUuid(); Import importUG = rootEm.get(importId, Import.class); importUG.setState(Import.State.FAILED); rootEm.update(importUG); } }
assertTrue( fileImport.getFileName().endsWith("testimport-bad-json.json")); fileImport.getErrorMessage().startsWith("Unexpected character ('<' (code 60))"));
final FileImport fileImport = new FileImport();
new Object[]{entities.size(), fileImport.getFileName()}); FileImport fi = (FileImport) itr.next(); switch (fi.getState()) { case FAILED: // failed, but we may not be complete so continue checking failCount++; default: // not something we recognize as complete, short circuit if (logger.isDebugEnabled()) { logger.debug("not done yet, bail out... {}", fi.getState().toString());
FileImport fileImport = new FileImport("dummyFileName" + i, applicationId); fileImport = emMgmtApp.create(fileImport); emMgmtApp.createConnection(importEntity, "includes", fileImport);
@Override public void doWrite(EntityManager em, FileImport fileImport, FileImportTracker tracker) { try { if (logger.isTraceEnabled()) { logger.trace("Writing imported entity {}:{} into app {}", entityType, entityUuid, em.getApplication().getUuid()); } em.create(entityUuid, entityType, properties); tracker.entityWritten(); } catch (Exception e) { logger.error("Error writing entity. From file:{}", fileImport.getFileName(), e); tracker.entityFailed(e.getMessage() + " From file: " + fileImport.getFileName()); } } }
@Override public void doWrite(EntityManager em, FileImport fileImport, FileImportTracker tracker) { try { // TODO: do we need to ensure that all Entity events happen first? // TODO: what happens if ConnectionEvents happen before all entities are saved? // Connections are specified as UUIDs with no type if (entityRef.getType() == null) { entityRef = em.get(ownerEntityRef.getUuid()); } if (logger.isTraceEnabled()) { logger.trace("Creating connection from {}:{} to {}:{}", ownerEntityRef.getType(), ownerEntityRef.getUuid(), entityRef.getType(), entityRef.getUuid()); } em.createConnection(ownerEntityRef, connectionType, entityRef); tracker.connectionWritten(); } catch (Exception e) { logger.error("Error writing connection. From file: {}", fileImport.getFileName(), e); tracker.connectionFailed(e.getMessage() + " From file: " + fileImport.getFileName()); } } }