private void createSecretManager() throws IOException { secretManager = new ShuffleSecretManager(); secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); fs.mkdirs(new Path(secretsFile.getPath()), new FsPermission((short)0700)); db = LevelDBProvider.initLevelDB(secretsFile, CURRENT_VERSION, mapper); logger.info("Recovery location is: " + secretsFile.getPath()); if (db != null) { logger.info("Going to reload spark shuffle data"); DBIterator itr = db.iterator(); itr.seek(APP_CREDS_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_CREDS_KEY_PREFIX)) { break; } String id = parseDbAppKey(key); ByteBuffer secret = mapper.readValue(e.getValue(), ByteBuffer.class); logger.info("Reloading tokens for app: " + id); secretManager.registerApp(id, secret); } } }
private void createSecretManager() throws IOException { secretManager = new ShuffleSecretManager(); secretsFile = initRecoveryDb(SECRETS_RECOVERY_FILE_NAME); // Make sure this is protected in case its not in the NM recovery dir FileSystem fs = FileSystem.getLocal(_conf); fs.mkdirs(new Path(secretsFile.getPath()), new FsPermission((short)0700)); db = LevelDBProvider.initLevelDB(secretsFile, CURRENT_VERSION, mapper); logger.info("Recovery location is: " + secretsFile.getPath()); if (db != null) { logger.info("Going to reload spark shuffle data"); DBIterator itr = db.iterator(); itr.seek(APP_CREDS_KEY_PREFIX.getBytes(StandardCharsets.UTF_8)); while (itr.hasNext()) { Map.Entry<byte[], byte[]> e = itr.next(); String key = new String(e.getKey(), StandardCharsets.UTF_8); if (!key.startsWith(APP_CREDS_KEY_PREFIX)) { break; } String id = parseDbAppKey(key); ByteBuffer secret = mapper.readValue(e.getValue(), ByteBuffer.class); logger.info("Reloading tokens for app: " + id); secretManager.registerApp(id, secret); } } }
boolean authEnabled = conf.getBoolean(SPARK_AUTHENTICATE_KEY, DEFAULT_SPARK_AUTHENTICATE); if (authEnabled) { secretManager = new ShuffleSecretManager(); if (_recoveryPath != null) { loadSecretsFromDb();
@Test public void testMultipleRegisters() { ShuffleSecretManager secretManager = new ShuffleSecretManager(); secretManager.registerApp(app1, pw1); assertEquals(pw1, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2.getBytes())); assertEquals(pw2, secretManager.getSecretKey(app2)); // now update the password for the apps and make sure it takes affect secretManager.registerApp(app1, pw1update); assertEquals(pw1update, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2update.getBytes())); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app1); assertNull(secretManager.getSecretKey(app1)); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app2); assertNull(secretManager.getSecretKey(app2)); assertNull(secretManager.getSecretKey(app1)); } }
@Test public void testMultipleRegisters() { ShuffleSecretManager secretManager = new ShuffleSecretManager(); secretManager.registerApp(app1, pw1); assertEquals(pw1, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2.getBytes())); assertEquals(pw2, secretManager.getSecretKey(app2)); // now update the password for the apps and make sure it takes affect secretManager.registerApp(app1, pw1update); assertEquals(pw1update, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2update.getBytes())); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app1); assertNull(secretManager.getSecretKey(app1)); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app2); assertNull(secretManager.getSecretKey(app2)); assertNull(secretManager.getSecretKey(app1)); } }
@Test public void testMultipleRegisters() { ShuffleSecretManager secretManager = new ShuffleSecretManager(); secretManager.registerApp(app1, pw1); assertEquals(pw1, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2.getBytes())); assertEquals(pw2, secretManager.getSecretKey(app2)); // now update the password for the apps and make sure it takes affect secretManager.registerApp(app1, pw1update); assertEquals(pw1update, secretManager.getSecretKey(app1)); secretManager.registerApp(app2, ByteBuffer.wrap(pw2update.getBytes())); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app1); assertNull(secretManager.getSecretKey(app1)); assertEquals(pw2update, secretManager.getSecretKey(app2)); secretManager.unregisterApp(app2); assertNull(secretManager.getSecretKey(app2)); assertNull(secretManager.getSecretKey(app1)); } }