PartitionTask(String dbName, String tblName, String authName, List<String> partNames) { this.dbName = safeIntern(dbName); this.tblName = safeIntern(tblName); this.authName = safeIntern(authName); this.partNames = partNames; }
/** * Retrieve a Hive full snapshot from HMS. * * @return HMS snapshot. Snapshot consists of a mapping from auth object name to the set of paths * corresponding to that name. */ private Map<String, Collection<String>> fetchFullUpdate() throws Exception{ String logMessage = "Request full HMS snapshot"; LOGGER.info(logMessage); System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage)); try (FullUpdateInitializer updateInitializer = new FullUpdateInitializer(hiveConnectionFactory, conf); Context context = updateTimer.time()) { SentryStateBank.enableState(FullUpdateInitializerState.COMPONENT,FullUpdateInitializerState.FULL_SNAPSHOT_INPROGRESS); Map<String, Collection<String>> pathsUpdate = updateInitializer.getFullHMSSnapshot(); logMessage = "Obtained full HMS snapshot"; LOGGER.info(logMessage); System.out.println(SentryServiceUtil.getCurrentTimeStampWithMessage(logMessage)); return pathsUpdate; } catch (Exception exception) { failedSnapshotsCount.inc(); LOGGER.error("Snapshot created failed ", exception); throw exception; } finally { SentryStateBank.disableState(FullUpdateInitializerState.COMPONENT,FullUpdateInitializerState.FULL_SNAPSHOT_INPROGRESS); } } }
String path = FullUpdateInitializer.pathFromURI(location); if (path == null) { return;
@Override ObjectMapping doTask() throws Exception { HMSClient c = null; try (HMSClient client = clientFactory.connect()) { c = client; Database db = client.getClient().getDatabase(dbName); if (!dbName.equalsIgnoreCase(db.getName())) { LOGGER.warn("Database name {} does not match {}", db.getName(), dbName); return emptyObjectMapping; } List<String> allTblStr = client.getClient().getAllTables(dbName); // Count total number of tables tableCount.inc(allTblStr.size()); for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) { List<String> tablesToFetch = allTblStr.subList(i, Math.min(i + maxTablesPerCall, allTblStr.size())); Callable<CallResult> tableTask = new TableTask(db, tablesToFetch); results.add(threadPool.submit(tableTask)); } String dbPath = safeIntern(pathFromURI(db.getLocationUri())); return (dbPath != null) ? new ObjectMapping(dbName, dbPath) : emptyObjectMapping; } catch (Exception e) { if (c != null) { c.invalidate(); } throw e; } } }
new FullUpdateInitializer(new MockHMSClientFactory(client), conf)) { update = cacheInitializer.getFullHMSSnapshot();
String path = FullUpdateInitializer.pathFromURI(location); if (path == null) { return;
String tableName = safeIntern(tbl.getTableName().toLowerCase()); String authzObject = (dbName + "." + tableName).intern(); List<String> tblPartNames = results.add(threadPool.submit(partTask)); String tblPath = safeIntern(pathFromURI(tbl.getSd().getLocation())); if (tblPath == null) { continue;
@Test // Test basic operation with small database public void testSimple() throws Exception { HiveTable tab21 = new HiveTable("tab21"); HiveTable tab31 = new HiveTable("tab31").add("part311").add("part312"); HiveDb db3 = new HiveDb("db3", Lists.newArrayList(tab31)); HiveDb db2 = new HiveDb("db2", Lists.newArrayList(tab21)); HiveDb db1 = new HiveDb("db1"); HiveSnapshot snap = new HiveSnapshot().add(db1).add(db2).add(db3); MockClient c = new MockClient(snap); Map<String, Collection<String>> update; try(FullUpdateInitializer cacheInitializer = new FullUpdateInitializer(new MockHMSClientFactory(c), conf)) { update = cacheInitializer.getFullHMSSnapshot(); } Assert.assertEquals(5, update.size()); Assert.assertEquals(Sets.newHashSet("db1"), update.get("db1")); Assert.assertEquals(Sets.newHashSet("db2"), update.get("db2")); Assert.assertEquals(Sets.newHashSet("db3"), update.get("db3")); Assert.assertEquals(Sets.newHashSet("db2/tab21"), update.get("db2.tab21")); Assert.assertEquals(Sets.newHashSet("db3/tab31", "db3/tab31/part311", "db3/tab31/part312"), update.get("db3.tab31")); }
String prevPath = FullUpdateInitializer.pathFromURI(prevLocation); if (prevPath == null) { return; String newPath = FullUpdateInitializer.pathFromURI(newLocation); if (newPath == null) { return;
DbTask(String dbName) { //Database names are case insensitive this.dbName = safeIntern(dbName.toLowerCase()); databaseCount.inc(); }
Map<String, Collection<String>> update; try(FullUpdateInitializer cacheInitializer = new FullUpdateInitializer(new MockHMSClientFactory(c), conf)) { update = cacheInitializer.getFullHMSSnapshot();
String path = FullUpdateInitializer.pathFromURI(location); if (path == null) { return;
ObjectMapping(String authObject, String path) { Set<String> values = Collections.singleton(safeIntern(path)); objects = ImmutableMap.of(authObject, values); }
String path = FullUpdateInitializer.pathFromURI(location); if (path == null) { return;
TableTask(Database db, List<String> tableNames) { dbName = safeIntern(db.getName()); this.tableNames = tableNames; }
String path = FullUpdateInitializer.pathFromURI(location); if (path != null) { LOGGER.debug("Adding partition {}:{}", authName, path);
String path = FullUpdateInitializer.pathFromURI(location); if (path != null) { oldLocations.remove(path);
return; String prevPath = FullUpdateInitializer.pathFromURI(prevLocation); if (prevPath == null) { return; return; String newPath = FullUpdateInitializer.pathFromURI(newLocation); if (newPath == null) { return;
@Override ObjectMapping doTask() throws Exception { List<Partition> tblParts; HMSClient c = null; try (HMSClient client = clientFactory.connect()) { c = client; tblParts = client.getClient().getPartitionsByNames(dbName, tblName, partNames); } catch (Exception e) { if (c != null) { c.invalidate(); } throw e; } LOGGER.debug("Fetched partitions for db = {}, table = {}", dbName, tblName); Collection<String> partitionNames = new ArrayList<>(tblParts.size()); for (Partition part : tblParts) { if(part != null && part.getSd() != null) { String partPath = pathFromURI(part.getSd().getLocation()); if (partPath != null) { partitionNames.add(partPath.intern()); } } else { LOGGER.info("Partition or its storage descriptor is null while fetching partitions for db = {} table = {}", dbName, tblName); } } return new ObjectMapping(authName, partitionNames); } }