/** * 1) Versioning timeline store: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, * 2.0 etc. 2) Any incompatible change of TS-store is a major upgrade, and any * compatible change of TS-store is a minor upgrade. 3) Within a minor * upgrade, say 1.1 to 1.2: overwrite the version info and proceed as normal. * 4) Within a major upgrade, say 1.2 to 2.0: throw exception and indicate * user to use a separate upgrade tool to upgrade timeline store or remove * incompatible old state. */ private void checkVersion() throws IOException { Version loadedVersion = loadVersion(); LOG.info("Loaded timeline store version info " + loadedVersion); if (loadedVersion.equals(getCurrentVersion())) { return; } if (loadedVersion.isCompatibleTo(getCurrentVersion())) { LOG.info("Storing timeline store version info " + getCurrentVersion()); dbStoreVersion(CURRENT_VERSION_INFO); } else { String incompatibleMessage = "Incompatible version for timeline store: " + "expecting version " + getCurrentVersion() + ", but loading version " + loadedVersion; LOG.error(incompatibleMessage); throw new IOException(incompatibleMessage); } }
/** * For a given key / value pair that has been written to the db, write * additional entries to the db for each primary filter. */ private static long writePrimaryFilterEntries(WriteBatch writeBatch, Map<String, Set<Object>> primaryFilters, byte[] key, byte[] value) throws IOException { long putCount = 0; if (primaryFilters != null) { for (Entry<String, Set<Object>> pf : primaryFilters.entrySet()) { for (Object pfval : pf.getValue()) { writeBatch.put(addPrimaryFilterToKey(pf.getKey(), pfval, key), value); ++putCount; } } } return putCount; }
starttimedb = factory.open(new File(starttimeDBPath.toString()), options); ownerdb = factory.open(new File(ownerDBPath.toString()), options); checkVersion(); startTimeWriteCache = Collections.synchronizedMap(new LRUMap( getStartTimeWriteCacheSize(conf))); startTimeReadCache = Collections.synchronizedMap(new LRUMap( getStartTimeReadCacheSize(conf)));
@Test public void testCheckVersion() throws IOException { RollingLevelDBTimelineStore dbStore = (RollingLevelDBTimelineStore) store; // default version Version defaultVersion = dbStore.getCurrentVersion(); Assert.assertEquals(defaultVersion, dbStore.loadVersion()); // compatible version Version compatibleVersion = Version.newInstance(defaultVersion.getMajorVersion(), defaultVersion.getMinorVersion() + 2); dbStore.storeVersion(compatibleVersion); Assert.assertEquals(compatibleVersion, dbStore.loadVersion()); restartTimelineStore(); dbStore = (RollingLevelDBTimelineStore) store; // overwrite the compatible version Assert.assertEquals(defaultVersion, dbStore.loadVersion()); // incompatible version Version incompatibleVersion = Version.newInstance(defaultVersion.getMajorVersion() + 1, defaultVersion.getMinorVersion()); dbStore.storeVersion(incompatibleVersion); try { restartTimelineStore(); Assert.fail("Incompatible version, should expect fail here."); } catch (ServiceStateException e) { Assert.assertTrue("Exception message mismatch", e.getMessage().contains("Incompatible version for timeline store")); } }
@Test public void testCacheSizes() { Configuration conf = new Configuration(); assertEquals(10000, RollingLevelDBTimelineStore.getStartTimeReadCacheSize(conf)); assertEquals(10000, RollingLevelDBTimelineStore.getStartTimeWriteCacheSize(conf)); conf.setInt( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE, 10001); assertEquals(10001, RollingLevelDBTimelineStore.getStartTimeReadCacheSize(conf)); conf = new Configuration(); conf.setInt( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE, 10002); assertEquals(10002, RollingLevelDBTimelineStore.getStartTimeWriteCacheSize(conf)); }
addPrimaryFilter(entity, key, prefixlen + PRIMARY_FILTERS_COLUMN.length); if (otherInfo) { Object o = null; String keyStr = parseRemainingKey(key, prefixlen + OTHER_INFO_COLUMN.length); try { addRelatedEntity(entity, key, prefixlen + RELATED_ENTITIES_COLUMN.length); TimelineEvent event = getEntityEvent(null, key, prefixlen + EVENTS_COLUMN.length, iterator.peekNext().getValue()); if (event != null) {
private void restartTimelineStore() throws IOException { // need to close so leveldb releases database lock if (store != null) { store.close(); } store = new RollingLevelDBTimelineStore(); store.init(config); store.start(); }
return checkStartTimeInDb(entity, startTime);
@Before public void setup() throws Exception { fsContext = FileContext.getLocalFSFileContext(); fsPath = new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsoluteFile(); fsContext.delete(new Path(fsPath.getAbsolutePath()), true); config.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH, fsPath.getAbsolutePath()); config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false); store = new RollingLevelDBTimelineStore(); store.init(config); store.start(); loadTestEntityData(); loadVerificationEntityData(); loadTestDomainData(); }