long t1 = System.currentTimeMillis(); try { List<String> entityTypes = getEntityTypes(); for (String entityType : entityTypes) { LeveldbIterator iterator = null; try { deleteLock.writeLock().lock(); iterator = getDbIterator(false); pfIterator = getDbIterator(false); while (deleteNextEntity(entityType, reverseTimestamp, iterator, pfIterator, seeked)) { typeCount++;
@Override public void run() { while (true) { long timestamp = System.currentTimeMillis() - ttl; try { discardOldEntities(timestamp); Thread.sleep(ttlInterval); } catch (IOException e) { LOG.error(e); } catch (InterruptedException e) { LOG.info("Deletion thread received interrupt, exiting"); break; } } } }
/** * 1) Versioning timeline store: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc. * 2) Any incompatible change of TS-store is a major upgrade, and any * compatible change of TS-store is a minor upgrade. * 3) Within a minor upgrade, say 1.1 to 1.2: * overwrite the version info and proceed as normal. * 4) Within a major upgrade, say 1.2 to 2.0: * throw exception and indicate user to use a separate upgrade tool to * upgrade timeline store or remove incompatible old state. */ private void checkVersion() throws IOException { Version loadedVersion = loadVersion(); LOG.info("Loaded timeline store version info " + loadedVersion); if (loadedVersion.equals(getCurrentVersion())) { return; } if (loadedVersion.isCompatibleTo(getCurrentVersion())) { LOG.info("Storing timeline store version info " + getCurrentVersion()); dbStoreVersion(CURRENT_VERSION_INFO); } else { String incompatibleMessage = "Incompatible version for timeline store: expecting version " + getCurrentVersion() + ", but loading version " + loadedVersion; LOG.error(incompatibleMessage); throw new IOException(incompatibleMessage); } }
private boolean deleteNextEntity(String entityType, byte[] ts) throws IOException, InterruptedException { LeveldbIterator iterator = null; LeveldbIterator pfIterator = null; try { iterator = ((LeveldbTimelineStore)store).getDbIterator(false); pfIterator = ((LeveldbTimelineStore)store).getDbIterator(false); return ((LeveldbTimelineStore)store).deleteNextEntity(entityType, ts, iterator, pfIterator, false); } catch(DBException e) { throw new IOException(e); } finally { IOUtils.cleanupWithLogger(null, iterator, pfIterator); } }
addPrimaryFilter(entity, key, prefixlen + PRIMARY_FILTERS_COLUMN.length); entity.addOtherInfo(parseRemainingKey(key, prefixlen + OTHER_INFO_COLUMN.length), GenericObjectMapper.read(iterator.peekNext().getValue())); addRelatedEntity(entity, key, prefixlen + RELATED_ENTITIES_COLUMN.length); if (events || (lastEvent && entity.getEvents().size() == 0)) { TimelineEvent event = getEntityEvent(null, key, prefixlen + EVENTS_COLUMN.length, iterator.peekNext().getValue()); if (event != null) {
checkVersion(); startTimeWriteCache = Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize( conf))); startTimeReadCache = Collections.synchronizedMap(new LRUMap(getStartTimeReadCacheSize( conf)));
primaryFilters, otherInfo, entities.get(2), domainId2); ((LeveldbTimelineStore)store).discardOldEntities(-123L); assertEquals(1, getEntitiesWithPrimaryFilter("type_1", pfPair).size()); assertEquals(3, getEntitiesWithPrimaryFilter("type_1", userFilter).size()); ((LeveldbTimelineStore)store).discardOldEntities(123L); assertEquals(0, getEntities("type_1").size()); assertEquals(0, getEntities("type_2").size()); assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size());
@Override public TimelinePutResponse put(TimelineEntities entities) { try { deleteLock.readLock().lock(); TimelinePutResponse response = new TimelinePutResponse(); for (TimelineEntity entity : entities.getEntities()) { put(entity, response, false); } return response; } finally { deleteLock.readLock().unlock(); } }
@Test public void testCheckVersion() throws IOException { LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store; // default version Version defaultVersion = dbStore.getCurrentVersion(); Assert.assertEquals(defaultVersion, dbStore.loadVersion()); // compatible version Version compatibleVersion = Version.newInstance(defaultVersion.getMajorVersion(), defaultVersion.getMinorVersion() + 2); dbStore.storeVersion(compatibleVersion); Assert.assertEquals(compatibleVersion, dbStore.loadVersion()); restartTimelineStore(); dbStore = (LeveldbTimelineStore) store; // overwrite the compatible version Assert.assertEquals(defaultVersion, dbStore.loadVersion()); // incompatible version Version incompatibleVersion = Version.newInstance( defaultVersion.getMajorVersion() + 1, defaultVersion.getMinorVersion()); dbStore.storeVersion(incompatibleVersion); try { restartTimelineStore(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for timeline store")); } }
@Test public void testCacheSizes() { Configuration conf = new Configuration(); assertEquals(10000, LeveldbTimelineStore.getStartTimeReadCacheSize(conf)); assertEquals(10000, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf)); conf.setInt( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE, 10001); assertEquals(10001, LeveldbTimelineStore.getStartTimeReadCacheSize(conf)); conf = new Configuration(); conf.setInt( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE, 10002); assertEquals(10002, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf)); }
Version loadVersion() throws IOException { try { byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY)); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. if (data == null || data.length == 0) { return getCurrentVersion(); } Version version = new VersionPBImpl(VersionProto.parseFrom(data)); return version; } catch(DBException e) { throw new IOException(e); } }
LeveldbIterator iterator = null; try { iterator = getDbIterator(false); List<String> entityTypes = new ArrayList<String>(); iterator.seek(ENTITY_ENTRY_PREFIX);
@Test public void testGetEntityTypes() throws IOException { List<String> entityTypes = ((LeveldbTimelineStore)store).getEntityTypes(); assertEquals(7, entityTypes.size()); assertEquals("ACL_ENTITY_TYPE_1", entityTypes.get(0)); assertEquals("OLD_ENTITY_TYPE_1", entityTypes.get(1)); assertEquals(entityType1, entityTypes.get(2)); assertEquals(entityType2, entityTypes.get(3)); assertEquals(entityType4, entityTypes.get(4)); assertEquals(entityType5, entityTypes.get(5)); }
addPrimaryFilter(entity, key, prefixlen + PRIMARY_FILTERS_COLUMN.length); entity.addOtherInfo(parseRemainingKey(key, prefixlen + OTHER_INFO_COLUMN.length), GenericObjectMapper.read(iterator.peekNext().getValue())); addRelatedEntity(entity, key, prefixlen + RELATED_ENTITIES_COLUMN.length); if (events || (lastEvent && entity.getEvents().size() == 0)) { TimelineEvent event = getEntityEvent(null, key, prefixlen + EVENTS_COLUMN.length, iterator.peekNext().getValue()); if (event != null) {
checkVersion(); startTimeWriteCache = Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize( conf))); startTimeReadCache = Collections.synchronizedMap(new LRUMap(getStartTimeReadCacheSize( conf)));
primaryFilters, otherInfo, entities.get(1), domainId2); ((LeveldbTimelineStore)store).discardOldEntities(0L); assertEquals(2, getEntities("type_1").size()); assertEquals(0, getEntities("type_2").size()); assertEquals(6, ((LeveldbTimelineStore)store).getEntityTypes().size()); ((LeveldbTimelineStore)store).discardOldEntities(123L); assertEquals(0, getEntities("type_1").size()); assertEquals(0, getEntities("type_2").size()); assertEquals(0, ((LeveldbTimelineStore)store).getEntityTypes().size()); assertEquals(0, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
@Override public TimelinePutResponse put(TimelineEntities entities) { try { deleteLock.readLock().lock(); TimelinePutResponse response = new TimelinePutResponse(); for (TimelineEntity entity : entities.getEntities()) { put(entity, response, false); } return response; } finally { deleteLock.readLock().unlock(); } }
Version loadVersion() throws IOException { try { byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY)); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. if (data == null || data.length == 0) { return getCurrentVersion(); } Version version = new VersionPBImpl(VersionProto.parseFrom(data)); return version; } catch(DBException e) { throw new IOException(e); } }
LeveldbIterator iterator = null; try { iterator = getDbIterator(false); List<String> entityTypes = new ArrayList<String>(); iterator.seek(ENTITY_ENTRY_PREFIX);
long t1 = System.currentTimeMillis(); try { List<String> entityTypes = getEntityTypes(); for (String entityType : entityTypes) { LeveldbIterator iterator = null; try { deleteLock.writeLock().lock(); iterator = getDbIterator(false); pfIterator = getDbIterator(false); while (deleteNextEntity(entityType, reverseTimestamp, iterator, pfIterator, seeked)) { typeCount++;