/** * Get the {@link IMetaStoreClient} with the provided name. * @throws IOException */ public IMetaStoreClient getClient(String name) throws IOException { if (!this.clients.containsKey(name)) { throw new IOException("There is no client with name " + name); } return this.clients.get(name).get(); }
/** * @return an auto returnable wrapper around a {@link IMetaStoreClient}. * @throws IOException * Note: if you must acquire multiple locks, please use {@link #safeGetClients} instead, as this call may deadlock. */ public AutoReturnableObject<IMetaStoreClient> getClient() throws IOException { return new AutoReturnableObject<>(this.pool); }
@VisibleForTesting public Optional<Partition> getPartitionObject(String completePartitionName) { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { List<String> partitionList = At_SPLITTER.splitToList(completePartitionName); if (partitionList.size() != 3) { log.warn("Invalid partition name " + completePartitionName); return Optional.<Partition>absent(); } Partition sourcePartition = client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2)); return Optional.fromNullable(sourcePartition); } catch (IOException | TException e) { log.warn("Unable to get partition object from metastore for partition " + completePartitionName); } return Optional.<Partition>absent(); }
/** * @return an auto returnable wrapper around a {@link IMetaStoreClient}. * @throws IOException * Note: if you must acquire multiple locks, please use {@link #safeGetClients} instead, as this call may deadlock. */ public AutoReturnableObject<IMetaStoreClient> getClient() throws IOException { return new AutoReturnableObject<>(this.pool); }
@VisibleForTesting public boolean dropPartition(String completePartitionName) { List<String> partitionList = At_SPLITTER.splitToList(completePartitionName); if (partitionList.size() != 3) { log.warn("Invalid partition name " + completePartitionName); return false; } try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { client.get().dropPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2), false); return true; } catch (IOException | TException e) { log.warn("Unable to drop Partition " + completePartitionName); } return false; }
@VisibleForTesting public boolean addPartition(Partition destPartition, String completePartitionName) { try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { client.get().add_partition(destPartition); return true; } catch (IOException | TException e) { log.warn("Unable to add Partition " + completePartitionName); } return false; }
public static HivePartitionDataset findDataset(String completePartitionName, State prop) throws IOException { synchronized (lock) { List<String> partitionList = AT_SPLITTER.splitToList(completePartitionName); Preconditions.checkArgument(partitionList.size() == 3, "Invalid partition name"); if (!pool.isPresent()) { pool = Optional.of(HiveMetastoreClientPool.get(new Properties(), Optional.fromNullable(new Properties().getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY)))); } try (AutoReturnableObject<IMetaStoreClient> client = pool.get().getClient()) { Table table = new Table(client.get().getTable(partitionList.get(0), partitionList.get(1))); Partition partition = new Partition(table, client.get().getPartition(partitionList.get(0), partitionList.get(1), partitionList.get(2))); return new HivePartitionDataset(partition); } catch (TException | HiveException e) { throw new IOException(e); } } }
@Override public boolean createDbIfNotExists(String dbName) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { return createDbIfNotExists(client.get(), dbName); } }
@Override public boolean existsTable(String dbName, String tableName) throws IOException { if (this.optimizedChecks && this.tableAndDbExistenceCache.getIfPresent(dbName + ":" + tableName ) != null ) { return true; } try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(TABLE_EXISTS).time()) { return client.get().tableExists(dbName, tableName); } } catch (TException e) { throw new IOException(String.format("Unable to check existence of table %s in db %s", tableName, dbName), e); } }
/** * Get all tables in db with given table pattern. */ public Collection<DbAndTable> getTables() throws IOException { List<DbAndTable> tables = Lists.newArrayList(); try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Iterable<String> databases = Iterables.filter(client.get().getAllDatabases(), new Predicate<String>() { @Override public boolean apply(String db) { return HiveDatasetFinder.this.whitelistBlacklist.acceptDb(db); } }); for (final String db : databases) { Iterable<String> tableNames = Iterables.filter(client.get().getAllTables(db), new Predicate<String>() { @Override public boolean apply(String table) { return HiveDatasetFinder.this.whitelistBlacklist.acceptTable(db, table); } }); for (String tableName : tableNames) { tables.add(new DbAndTable(db, tableName)); } } } catch (Exception exc) { throw new IOException(exc); } return tables; }
private static List<Partition> getPartitions(String completeTableName) { List<String> tableList = At_SPLITTER.splitToList(completeTableName); if (tableList.size() != 2) { log.warn("Invalid table name " + completeTableName); return Collections.EMPTY_LIST; } try (AutoReturnableObject<IMetaStoreClient> client = ComplianceRetentionJob.pool.getClient()) { Table table = client.get().getTable(tableList.get(0), tableList.get(1)); HiveDataset dataset = new HiveDataset(FileSystem.newInstance(new Configuration()), ComplianceRetentionJob.pool, new org.apache.hadoop.hive.ql.metadata.Table(table), new Properties()); return dataset.getPartitionsFromDataset(); } catch (IOException | TException e) { log.warn("Unable to get Partitions for table " + completeTableName + " " + e.getMessage()); } return Collections.EMPTY_LIST; } }
@Override public boolean existsPartition(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { try (Timer.Context context = this.metricContext.timer(GET_HIVE_PARTITION).time()) { client.get().getPartition(dbName, tableName, partitionValues); } return true; } catch (NoSuchObjectException e) { return false; } catch (TException e) { throw new IOException(String.format("Unable to check existence of partition %s in table %s in db %s", partitionValues, tableName, dbName), e); } }
@Override public void run() throws Exception { Iterator<HiveDataset> iterator = this.datasetFinder.getDatasetsIterator(); while (iterator.hasNext()) { ConvertibleHiveDataset hiveDataset = (ConvertibleHiveDataset) iterator.next(); try (AutoReturnableObject<IMetaStoreClient> client = hiveDataset.getClientPool().getClient()) { Set<Partition> sourcePartitions = new HashSet<>(HiveUtils.getPartitions(client.get(), hiveDataset.getTable(), Optional.<String>absent())); sourcePartitions.parallelStream().filter(partition -> isUnixTimeStamp(partition.getDataLocation().getName())) .forEach(partition -> { Arrays.stream(listFiles(partition.getDataLocation().getParent())).filter( fileStatus -> !fileStatus.getPath().toString() .equalsIgnoreCase(partition.getDataLocation().toString())).forEach(fileStatus -> { deletePath(fileStatus, this.graceTimeInMillis, true); }); }); } } }
@Override public Optional<HiveTable> getTable(String dbName, String tableName) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Table hiveTable; try (Timer.Context context = this.metricContext.timer(GET_HIVE_TABLE).time()) { hiveTable = client.get().getTable(dbName, tableName); } return Optional.of(HiveMetaStoreUtils.getHiveTable(hiveTable)); } catch (NoSuchObjectException e) { return Optional.<HiveTable> absent(); } catch (TException e) { throw new IOException("Unable to get table " + tableName + " in db " + dbName, e); } }
public HiveMetadataForCompactionExtractor(WorkUnitState state, FileSystem fs) throws IOException, TException, HiveException { super(state); if (Boolean.valueOf(state.getPropAsBoolean(PartitionLevelWatermarker.IS_WATERMARK_WORKUNIT_KEY))) { log.info("Ignoring Watermark workunit for {}", state.getProp(ConfigurationKeys.DATASET_URN_KEY)); return; } try (AutoReturnableObject<IMetaStoreClient> client = this.pool.getClient()) { Table table = client.get().getTable(this.dbName, this.tableName); String primaryKeyString = table.getParameters().get(state.getProp(COMPACTION_PRIMARY_KEY)); List<String> primaryKeyList = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(primaryKeyString); String deltaString = table.getParameters().get(state.getProp(COMPACTION_DELTA)); List<String> deltaList = Splitter.on(',').omitEmptyStrings().trimResults().splitToList(deltaString); Path dataFilesPath = new Path(table.getSd().getLocation()); compactionEntity = new MRCompactionEntity(primaryKeyList, deltaList, dataFilesPath, state.getProperties()); } }
@Override public Optional<HivePartition> getPartition(String dbName, String tableName, List<Column> partitionKeys, List<String> partitionValues) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Partition hivePartition; try (Timer.Context context = this.metricContext.timer(GET_HIVE_PARTITION).time()) { hivePartition = client.get().getPartition(dbName, tableName, partitionValues); } return Optional.of(HiveMetaStoreUtils.getHivePartition(hivePartition)); } catch (NoSuchObjectException e) { return Optional.<HivePartition> absent(); } catch (TException e) { throw new IOException( "Unable to get partition " + partitionValues + " from table " + tableName + " in db " + dbName, e); } }
Optional.fromNullable(props.getProperty(HiveDatasetFinder.HIVE_METASTORE_URI_KEY))); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { table = Optional.of(client.get().getTable(dbName, tableName)); if (table.isPresent()) { org.apache.hadoop.hive.ql.metadata.Table qlTable = new org.apache.hadoop.hive.ql.metadata.Table(table.get()); if (HiveUtils.isPartitioned(qlTable)) { partitions = Optional.of(HiveUtils.getPartitions(client.get(), qlTable, Optional.<String>absent()));
@Override public void addOrAlterPartition(HiveTable table, HivePartition partition) throws IOException { try (AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { addOrAlterPartition(client.get(), HiveMetaStoreUtils.getTable(table), partition); } catch (TException te) { throw new IOException( String.format("Failed to add/alter partition %s.%s@%s", table.getDbName(), table.getTableName(), partition.getValues()), te); } }
@Override protected void registerPath(HiveSpec spec) throws IOException { try (Timer.Context context = this.metricContext.timer(PATH_REGISTER_TIMER).time(); AutoReturnableObject<IMetaStoreClient> client = this.clientPool.getClient()) { Table table = HiveMetaStoreUtils.getTable(spec.getTable()); createDbIfNotExists(client.get(), table.getDbName()); createOrAlterTable(client.get(), table, spec); Optional<HivePartition> partition = spec.getPartition(); if (partition.isPresent()) { addOrAlterPartition(client.get(), table, partition.get()); } HiveMetaStoreEventHelper.submitSuccessfulPathRegistration(eventSubmitter, spec); } catch (TException e) { HiveMetaStoreEventHelper.submitFailedPathRegistration(eventSubmitter, spec, e); throw new IOException(e); } }
List<Partition> partitions = HiveUtils.getPartitions(client.get(), hiveDataset.getTable(), Optional.<String> absent()); return Lists.newArrayList(Iterables.filter(Iterables.transform(partitions, new Function<Partition, HiveDatasetVersion>() {