/** * 1) Versioning timeline state store: * major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc. * 2) Any incompatible change of TS-store is a major upgrade, and any * compatible change of TS-store is a minor upgrade. * 3) Within a minor upgrade, say 1.1 to 1.2: * overwrite the version info and proceed as normal. * 4) Within a major upgrade, say 1.2 to 2.0: * throw exception and indicate user to use a separate upgrade tool to * upgrade timeline store or remove incompatible old state. */ private void checkVersion() throws IOException { Version loadedVersion = loadVersion(); LOG.info("Loaded timeline state store version info " + loadedVersion); if (loadedVersion.equals(getCurrentVersion())) { return; } if (loadedVersion.isCompatibleTo(getCurrentVersion())) { LOG.info("Storing timeline state store version info " + getCurrentVersion()); storeVersion(CURRENT_VERSION_INFO); } else { String incompatibleMessage = "Incompatible version for timeline state store: expecting version " + getCurrentVersion() + ", but loading version " + loadedVersion; LOG.error(incompatibleMessage); throw new IOException(incompatibleMessage); } }
@Override public void updateToken(TimelineDelegationTokenIdentifier tokenId, Long renewDate) throws IOException { try { byte[] k = createTokenEntryKey(tokenId.getSequenceNumber()); if (db.get(k) == null) { throw new IOException(tokenId + " doesn't exist"); } byte[] v = buildTokenData(tokenId, renewDate); db.put(k, v); } catch (DBException e) { throw new IOException(e); } }
@Override public void storeTokenMasterKey(DelegationKey key) throws IOException { try { byte[] k = createTokenMasterKeyEntryKey(key.getKeyId()); if (db.get(k) != null) { throw new IOException(key + " already exists"); } byte[] v = buildTokenMasterKeyData(key); db.put(k, v); } catch (DBException e) { throw new IOException(e); } }
@Override public TimelineServiceState loadState() throws IOException { LOG.info("Loading timeline service state from leveldb"); TimelineServiceState state = new TimelineServiceState(); int numKeys = loadTokenMasterKeys(state); int numTokens = loadTokens(state); loadLatestSequenceNumber(state); LOG.info("Loaded " + numKeys + " master keys and " + numTokens + " tokens from leveldb, and latest sequence number is " + state.getLatestSequenceNumber()); return state; }
initAndStartTimelineServiceStateStoreService(); Version defaultVersion = store.getCurrentVersion(); Assert.assertEquals(defaultVersion, store.loadVersion()); Version.newInstance(defaultVersion.getMajorVersion(), defaultVersion.getMinorVersion() + 2); store.storeVersion(compatibleVersion); Assert.assertEquals(compatibleVersion, store.loadVersion()); store.stop(); Assert.assertEquals(defaultVersion, store.loadVersion()); Version.newInstance(defaultVersion.getMajorVersion() + 1, defaultVersion.getMinorVersion()); store.storeVersion(incompatibleVersion); store.stop();
Path dbPath = new Path( getConfig().get( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH), DB_NAME); FileSystem localFS = null; try { localFS = FileSystem.getLocal(getConfig()); if (!localFS.exists(dbPath)) { if (!localFS.mkdirs(dbPath)) { db = factory.open(new File(dbPath.toString()), options); LOG.info("Loading the existing database at th path: " + dbPath.toString()); checkVersion(); } catch (NativeDB.DBException e) { if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { db = factory.open(new File(dbPath.toString()), options); LOG.info("Creating a new database at th path: " + dbPath.toString()); storeVersion(CURRENT_VERSION_INFO); } catch (DBException ex) { throw new IOException(ex);
@VisibleForTesting Version loadVersion() throws IOException { try { byte[] data = db.get(TIMELINE_STATE_STORE_VERSION_KEY); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. if (data == null || data.length == 0) { return getCurrentVersion(); } Version version = new VersionPBImpl( YarnServerCommonProtos.VersionProto.parseFrom(data)); return version; } catch (DBException e) { throw new IOException(e); } }
@Override public void removeTokenMasterKey(DelegationKey key) throws IOException { try { byte[] k = createTokenMasterKeyEntryKey(key.getKeyId()); db.delete(k); } catch (DBException e) { throw new IOException(e); } }
@Override public void removeToken(TimelineDelegationTokenIdentifier tokenId) throws IOException { try { byte[] key = createTokenEntryKey(tokenId.getSequenceNumber()); db.delete(key); } catch (DBException e) { throw new IOException(e); } }
Path dbPath = new Path( getConfig().get( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH), DB_NAME); FileSystem localFS = null; try { localFS = FileSystem.getLocal(getConfig()); if (!localFS.exists(dbPath)) { if (!localFS.mkdirs(dbPath)) { db = factory.open(new File(dbPath.toString()), options); LOG.info("Loading the existing database at th path: " + dbPath.toString()); checkVersion(); } catch (NativeDB.DBException e) { if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { db = factory.open(new File(dbPath.toString()), options); LOG.info("Creating a new database at th path: " + dbPath.toString()); storeVersion(CURRENT_VERSION_INFO); } catch (DBException ex) { throw new IOException(ex);
@Override public TimelineServiceState loadState() throws IOException { LOG.info("Loading timeline service state from leveldb"); TimelineServiceState state = new TimelineServiceState(); int numKeys = loadTokenMasterKeys(state); int numTokens = loadTokens(state); loadLatestSequenceNumber(state); LOG.info("Loaded " + numKeys + " master keys and " + numTokens + " tokens from leveldb, and latest sequence number is " + state.getLatestSequenceNumber()); return state; }
@VisibleForTesting Version loadVersion() throws IOException { try { byte[] data = db.get(TIMELINE_STATE_STORE_VERSION_KEY); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. if (data == null || data.length == 0) { return getCurrentVersion(); } Version version = new VersionPBImpl( YarnServerCommonProtos.VersionProto.parseFrom(data)); return version; } catch (DBException e) { throw new IOException(e); } }
@Override public void removeTokenMasterKey(DelegationKey key) throws IOException { try { byte[] k = createTokenMasterKeyEntryKey(key.getKeyId()); db.delete(k); } catch (DBException e) { throw new IOException(e); } }
@Override public void removeToken(TimelineDelegationTokenIdentifier tokenId) throws IOException { try { byte[] key = createTokenEntryKey(tokenId.getSequenceNumber()); db.delete(key); } catch (DBException e) { throw new IOException(e); } }
/** * 1) Versioning timeline state store: * major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc. * 2) Any incompatible change of TS-store is a major upgrade, and any * compatible change of TS-store is a minor upgrade. * 3) Within a minor upgrade, say 1.1 to 1.2: * overwrite the version info and proceed as normal. * 4) Within a major upgrade, say 1.2 to 2.0: * throw exception and indicate user to use a separate upgrade tool to * upgrade timeline store or remove incompatible old state. */ private void checkVersion() throws IOException { Version loadedVersion = loadVersion(); LOG.info("Loaded timeline state store version info " + loadedVersion); if (loadedVersion.equals(getCurrentVersion())) { return; } if (loadedVersion.isCompatibleTo(getCurrentVersion())) { LOG.info("Storing timeline state store version info " + getCurrentVersion()); storeVersion(CURRENT_VERSION_INFO); } else { String incompatibleMessage = "Incompatible version for timeline state store: expecting version " + getCurrentVersion() + ", but loading version " + loadedVersion; LOG.fatal(incompatibleMessage); throw new IOException(incompatibleMessage); } }
Path dbPath = new Path( getConfig().get( YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH), DB_NAME); FileSystem localFS = null; try { localFS = FileSystem.getLocal(getConfig()); if (!localFS.exists(dbPath)) { if (!localFS.mkdirs(dbPath)) { db = factory.open(new File(dbPath.toString()), options); LOG.info("Loading the existing database at th path: " + dbPath.toString()); checkVersion(); } catch (NativeDB.DBException e) { if (e.isNotFound() || e.getMessage().contains(" does not exist ")) { db = factory.open(new File(dbPath.toString()), options); LOG.info("Creating a new database at th path: " + dbPath.toString()); storeVersion(CURRENT_VERSION_INFO); } catch (DBException ex) { throw new IOException(ex);
@Override public TimelineServiceState loadState() throws IOException { LOG.info("Loading timeline service state from leveldb"); TimelineServiceState state = new TimelineServiceState(); int numKeys = loadTokenMasterKeys(state); int numTokens = loadTokens(state); loadLatestSequenceNumber(state); LOG.info("Loaded " + numKeys + " master keys and " + numTokens + " tokens from leveldb, and latest sequence number is " + state.getLatestSequenceNumber()); return state; }
@Override public void updateToken(TimelineDelegationTokenIdentifier tokenId, Long renewDate) throws IOException { try { byte[] k = createTokenEntryKey(tokenId.getSequenceNumber()); if (db.get(k) == null) { throw new IOException(tokenId + " doesn't exist"); } byte[] v = buildTokenData(tokenId, renewDate); db.put(k, v); } catch (DBException e) { throw new IOException(e); } }
@Override public void storeTokenMasterKey(DelegationKey key) throws IOException { try { byte[] k = createTokenMasterKeyEntryKey(key.getKeyId()); if (db.get(k) != null) { throw new IOException(key + " already exists"); } byte[] v = buildTokenMasterKeyData(key); db.put(k, v); } catch (DBException e) { throw new IOException(e); } }
@VisibleForTesting Version loadVersion() throws IOException { try { byte[] data = db.get(TIMELINE_STATE_STORE_VERSION_KEY); // if version is not stored previously, treat it as CURRENT_VERSION_INFO. if (data == null || data.length == 0) { return getCurrentVersion(); } Version version = new VersionPBImpl( YarnServerCommonProtos.VersionProto.parseFrom(data)); return version; } catch (DBException e) { throw new IOException(e); } }