/** * Creates a pair. Constructor doesn't infer template args but * the method does, so the code becomes less ugly. */ public static <T1, T2> ObjectPair<T1, T2> create(T1 f, T2 s) { return new ObjectPair<>(f, s); }
@Override public boolean equals(Object that) { if (that == null) { return false; } if (that instanceof ObjectPair) { return this.equals((ObjectPair<F, S>)that); } return false; }
@SuppressWarnings("unchecked") private <T> List<T> queryByPartitionNames(String catName, String dbName, String tableName, List<String> partNames, Class<T> clazz, String tbCol, String dbCol, String partCol, String catCol) { ObjectPair<Query, Object[]> queryAndParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames, clazz, tbCol, dbCol, partCol, catCol); return (List<T>)queryAndParams.getFirst().executeWithArray(queryAndParams.getSecond()); }
private ObjectPair<Query, Map<String, String>> getPartQueryWithParams( String catName, String dbName, String tblName, List<String> partNames) { Query query = pm.newQuery(); Map<String, String> params = new HashMap<>(); String filterStr = getJDOFilterStrForPartitionNames(catName, dbName, tblName, partNames, params); query.setFilter(filterStr); LOG.debug(" JDOQL filter is {}", filterStr); query.declareParameters(makeParameterDeclarationString(params)); return new ObjectPair<>(query, params); }
private void dropPartitionColumnStatisticsNoTxn( String catName, String dbName, String tableName, List<String> partNames) throws MetaException { ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames( catName, dbName, tableName, partNames, MPartitionColumnStatistics.class, "tableName", "dbName", "partition.partitionName", "catName"); queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); }
private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec, boolean ifExists, boolean deleteData) throws SemanticException, TException { LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions."); ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); ObjectPair<Integer, byte[]> serializedPartitionExpression = new ObjectPair<>(partitionSpec.size(), SerializationUtilities.serializeExpressionToKryo(partitionExpression)); hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression), deleteData && !isExternal(table), // Delete data? ifExists, // Fail if table doesn't exist? false); // Need results back? }
private void dropPartitionAllColumnGrantsNoTxn( String catName, String dbName, String tableName, List<String> partNames) { ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames, MPartitionColumnPrivilege.class, "partition.table.tableName", "partition.table.database.name", "partition.partitionName", "partition.table.database.catalogName"); queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); }
public List<Partition> dropPartitions(String dbName, String tblName, List<DropTableDesc.PartSpec> partSpecs, PartitionDropOptions dropOptions) throws HiveException { try { Table tbl = getTable(dbName, tblName); List<org.apache.hadoop.hive.metastore.utils.ObjectPair<Integer, byte[]>> partExprs = new ArrayList<>(partSpecs.size()); for (DropTableDesc.PartSpec partSpec : partSpecs) { partExprs.add(new org.apache.hadoop.hive.metastore.utils.ObjectPair<>(partSpec.getPrefixLength(), SerializationUtilities.serializeExpressionToKryo(partSpec.getPartSpec()))); } List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().dropPartitions( dbName, tblName, partExprs, dropOptions); return convertFromMetastore(tbl, tParts); } catch (NoSuchObjectException e) { throw new HiveException("Partition or table doesn't exist.", e); } catch (Exception e) { throw new HiveException(e.getMessage(), e); } }
private void dropPartitionGrantsNoTxn(String catName, String dbName, String tableName, List<String> partNames) { ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames,MPartitionPrivilege.class, "partition.table.tableName", "partition.table.database.name", "partition.partitionName", "partition.table.database.catalogName"); queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); }
private ObjectPair<Query, Object[]> makeQueryByPartitionNames( String catName, String dbName, String tableName, List<String> partNames, Class<?> clazz, String tbCol, String dbCol, String partCol, String catCol) { String queryStr = tbCol + " == t1 && " + dbCol + " == t2 && " + catCol + " == t3"; String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; Object[] params = new Object[3 + partNames.size()]; params[0] = normalizeIdentifier(tableName); params[1] = normalizeIdentifier(dbName); params[2] = normalizeIdentifier(catName); int index = 0; for (String partName : partNames) { params[index + 3] = partName; queryStr += ((index == 0) ? " && (" : " || ") + partCol + " == p" + index; paramStr += ", java.lang.String p" + index; ++index; } queryStr += ")"; Query query = pm.newQuery(clazz, queryStr); query.declareParameters(paramStr); return new ObjectPair<>(query, params); }
private void dropPartitionsNoTxn(String catName, String dbName, String tblName, List<String> partNames) { ObjectPair<Query, Map<String, String>> queryWithParams = getPartQueryWithParams(catName, dbName, tblName, partNames); Query query = queryWithParams.getFirst(); query.setClass(MPartition.class); long deleted = query.deletePersistentAll(queryWithParams.getSecond()); LOG.debug("Deleted {} partition from store", deleted); query.closeAll(); }
private ObjectPair<Query, Map<String, String>> getPartQueryWithParams( String catName, String dbName, String tblName, List<String> partNames) { StringBuilder sb = new StringBuilder("table.tableName == t1 && table.database.name == t2 &&" + " table.database.catalogName == t3 && ("); int n = 0; Map<String, String> params = new HashMap<>(); for (Iterator<String> itr = partNames.iterator(); itr.hasNext();) { String pn = "p" + n; n++; String part = itr.next(); params.put(pn, part); sb.append("partitionName == ").append(pn); sb.append(" || "); } sb.setLength(sb.length() - 4); // remove the last " || " sb.append(')'); Query query = pm.newQuery(); query.setFilter(sb.toString()); LOG.debug(" JDOQL filter is {}", sb); params.put("t1", normalizeIdentifier(tblName)); params.put("t2", normalizeIdentifier(dbName)); params.put("t3", normalizeIdentifier(catName)); query.declareParameters(makeParameterDeclarationString(params)); return new ObjectPair<>(query, params); }
/** * Gets partition names from the table via ORM (JDOQL) name filter. * @param dbName Database name. * @param tblName Table name. * @param partNames Partition names to get the objects for. * @return Resulting partitions. */ private List<Partition> getPartitionsViaOrmFilter(String catName, String dbName, String tblName, List<String> partNames) throws MetaException { if (partNames.isEmpty()) { return new ArrayList<>(); } ObjectPair<Query, Map<String, String>> queryWithParams = getPartQueryWithParams(catName, dbName, tblName, partNames); Query query = queryWithParams.getFirst(); query.setResultClass(MPartition.class); query.setClass(MPartition.class); query.setOrdering("partitionName ascending"); @SuppressWarnings("unchecked") List<MPartition> mparts = (List<MPartition>)query.executeWithMap(queryWithParams.getSecond()); List<Partition> partitions = convertToParts(catName, dbName, tblName, mparts); if (query != null) { query.closeAll(); } return partitions; }
private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec, boolean ifExists, boolean deleteData) throws SemanticException, TException { LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions."); ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); ObjectPair<Integer, byte[]> serializedPartitionExpression = new ObjectPair<>(partitionSpec.size(), SerializationUtilities.serializeExpressionToKryo(partitionExpression)); hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression), deleteData && !isExternal(table), // Delete data? ifExists, // Fail if table doesn't exist? false); // Need results back? }
/** * Detaches column descriptors from storage descriptors; returns the set of unique CDs * thus detached. This is done before dropping partitions because CDs are reused between * SDs; so, we remove the links to delete SDs and then check the returned CDs to see if * they are referenced by other SDs. */ private HashSet<MColumnDescriptor> detachCdsFromSdsNoTxn( String catName, String dbName, String tblName, List<String> partNames) { ObjectPair<Query, Map<String, String>> queryWithParams = getPartQueryWithParams(catName, dbName, tblName, partNames); Query query = queryWithParams.getFirst(); query.setClass(MPartition.class); query.setResult("sd"); @SuppressWarnings("unchecked") List<MStorageDescriptor> sds = (List<MStorageDescriptor>)query.executeWithMap( queryWithParams.getSecond()); HashSet<MColumnDescriptor> candidateCds = new HashSet<>(); for (MStorageDescriptor sd : sds) { if (sd != null && sd.getCD() != null) { candidateCds.add(sd.getCD()); sd.setCD(null); } } if (query != null) { query.closeAll(); } return candidateCds; }
private ObjectPair<Query, Object[]> makeQueryByPartitionNames( String catName, String dbName, String tableName, List<String> partNames, Class<?> clazz, String tbCol, String dbCol, String partCol, String catCol) { String queryStr = tbCol + " == t1 && " + dbCol + " == t2 && " + catCol + " == t3"; String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; Object[] params = new Object[3 + partNames.size()]; params[0] = normalizeIdentifier(tableName); params[1] = normalizeIdentifier(dbName); params[2] = normalizeIdentifier(catName); int index = 0; for (String partName : partNames) { params[index + 3] = partName; queryStr += ((index == 0) ? " && (" : " || ") + partCol + " == p" + index; paramStr += ", java.lang.String p" + index; ++index; } queryStr += ")"; Query query = pm.newQuery(clazz, queryStr); query.declareParameters(paramStr); return new ObjectPair<>(query, params); }
@Override public List<Partition> dropPartitions(String catName, String dbName, String tblName, List<ObjectPair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws TException { RequestPartsSpec rps = new RequestPartsSpec(); List<DropPartitionsExpr> exprs = new ArrayList<>(partExprs.size()); for (ObjectPair<Integer, byte[]> partExpr : partExprs) { DropPartitionsExpr dpe = new DropPartitionsExpr(); dpe.setExpr(partExpr.getSecond()); dpe.setPartArchiveLevel(partExpr.getFirst()); exprs.add(dpe); } rps.setExprs(exprs); DropPartitionsRequest req = new DropPartitionsRequest(dbName, tblName, rps); req.setCatName(catName); req.setDeleteData(options.deleteData); req.setNeedResult(options.returnResults); req.setIfExists(options.ifExists); if (options.purgeData) { LOG.info("Dropped partitions will be purged!"); req.setEnvironmentContext(getEnvironmentContextWithIfPurgeSet()); } return client.drop_partitions_req(req).getPartitions(); }
@Override public List<Partition> dropPartitions(String dbName, String tblName, List<ObjectPair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws TException { RequestPartsSpec rps = new RequestPartsSpec(); List<DropPartitionsExpr> exprs = new ArrayList<>(partExprs.size()); for (ObjectPair<Integer, byte[]> partExpr : partExprs) { DropPartitionsExpr dpe = new DropPartitionsExpr(); dpe.setExpr(partExpr.getSecond()); dpe.setPartArchiveLevel(partExpr.getFirst()); exprs.add(dpe); } rps.setExprs(exprs); DropPartitionsRequest req = new DropPartitionsRequest(dbName, tblName, rps); req.setDeleteData(options.deleteData); req.setNeedResult(options.returnResults); req.setIfExists(options.ifExists); if (options.purgeData) { LOG.info("Dropped partitions will be purged!"); req.setEnvironmentContext(getEnvironmentContextWithIfPurgeSet()); } return client.drop_partitions_req(req).getPartitions(); }
private void dropPartitionAllColumnGrantsNoTxn( String catName, String dbName, String tableName, List<String> partNames) { ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames, MPartitionColumnPrivilege.class, "partition.table.tableName", "partition.table.database.name", "partition.partitionName", "partition.table.database.catalogName"); queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); }