public static String serializeExpression(ExprNodeGenericFuncDesc expr) { return new String(Base64.encodeBase64(serializeExpressionToKryo(expr)), StandardCharsets.UTF_8); }
public static String serializeExpression(ExprNodeGenericFuncDesc expr) { try { return new String(Base64.encodeBase64(serializeExpressionToKryo(expr)), "UTF-8"); } catch (UnsupportedEncodingException ex) { throw new RuntimeException("UTF-8 support required", ex); } }
private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec, boolean ifExists, boolean deleteData) throws SemanticException, TException { LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions."); ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); ObjectPair<Integer, byte[]> serializedPartitionExpression = new ObjectPair<>(partitionSpec.size(), SerializationUtilities.serializeExpressionToKryo(partitionExpression)); hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression), deleteData && !isExternal(table), // Delete data? ifExists, // Fail if table doesn't exist? false); // Need results back? }
/** * Get a list of Partitions by expr. * @param tbl The table containing the partitions. * @param expr A serialized expression for partition predicates. * @param conf Hive config. * @param result the resulting list of partitions * @return whether the resulting list contains partitions which may or may not match the expr */ public boolean getPartitionsByExpr(Table tbl, ExprNodeGenericFuncDesc expr, HiveConf conf, List<Partition> result) throws HiveException, TException { assert result != null; byte[] exprBytes = SerializationUtilities.serializeExpressionToKryo(expr); String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME); List<org.apache.hadoop.hive.metastore.api.Partition> msParts = new ArrayList<org.apache.hadoop.hive.metastore.api.Partition>(); boolean hasUnknownParts = getMSC().listPartitionsByExpr(tbl.getDbName(), tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts); result.addAll(convertFromMetastore(tbl, msParts)); return hasUnknownParts; }
public List<Partition> dropPartitions(String dbName, String tblName, List<DropTableDesc.PartSpec> partSpecs, PartitionDropOptions dropOptions) throws HiveException { try { Table tbl = getTable(dbName, tblName); List<org.apache.hadoop.hive.metastore.utils.ObjectPair<Integer, byte[]>> partExprs = new ArrayList<>(partSpecs.size()); for (DropTableDesc.PartSpec partSpec : partSpecs) { partExprs.add(new org.apache.hadoop.hive.metastore.utils.ObjectPair<>(partSpec.getPrefixLength(), SerializationUtilities.serializeExpressionToKryo(partSpec.getPartSpec()))); } List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().dropPartitions( dbName, tblName, partExprs, dropOptions); return convertFromMetastore(tbl, tParts); } catch (NoSuchObjectException e) { throw new HiveException("Partition or table doesn't exist.", e); } catch (Exception e) { throw new HiveException(e.getMessage(), e); } }
public List<Partition> dropPartitions(String dbName, String tblName, List<DropTableDesc.PartSpec> partSpecs, PartitionDropOptions dropOptions) throws HiveException { try { Table tbl = getTable(dbName, tblName); List<ObjectPair<Integer, byte[]>> partExprs = new ArrayList<ObjectPair<Integer,byte[]>>(partSpecs.size()); for (DropTableDesc.PartSpec partSpec : partSpecs) { partExprs.add(new ObjectPair<Integer, byte[]>(partSpec.getPrefixLength(), SerializationUtilities.serializeExpressionToKryo(partSpec.getPartSpec()))); } List<org.apache.hadoop.hive.metastore.api.Partition> tParts = getMSC().dropPartitions( dbName, tblName, partExprs, dropOptions); return convertFromMetastore(tbl, tParts); } catch (NoSuchObjectException e) { throw new HiveException("Partition or table doesn't exist.", e); } catch (Exception e) { throw new HiveException(e.getMessage(), e); } }
/** * Get a list of Partitions by expr. * @param tbl The table containing the partitions. * @param expr A serialized expression for partition predicates. * @param conf Hive config. * @param result the resulting list of partitions * @return whether the resulting list contains partitions which may or may not match the expr */ public boolean getPartitionsByExpr(Table tbl, ExprNodeGenericFuncDesc expr, HiveConf conf, List<Partition> result) throws HiveException, TException { assert result != null; byte[] exprBytes = SerializationUtilities.serializeExpressionToKryo(expr); String defaultPartitionName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME); List<org.apache.hadoop.hive.metastore.api.Partition> msParts = new ArrayList<org.apache.hadoop.hive.metastore.api.Partition>(); boolean hasUnknownParts = getMSC().listPartitionsByExpr(tbl.getDbName(), tbl.getTableName(), exprBytes, defaultPartitionName, (short)-1, msParts); result.addAll(convertFromMetastore(tbl, msParts)); return hasUnknownParts; }
public void checkExpr(int numParts, String dbName, String tblName, ExprNodeGenericFuncDesc expr) throws Exception { List<Partition> parts = new ArrayList<Partition>(); client.listPartitionsByExpr(dbName, tblName, SerializationUtilities.serializeExpressionToKryo(expr), null, (short)-1, parts); assertEquals("Partition check failed: " + expr.getExprString(), numParts, parts.size()); }
public static List<ObjectPair<Integer, byte[]>> toObjectPairs(Table table, List<Partition> partitions) { List<ObjectPair<Integer, byte[]>> pairs = new ArrayList<>(partitions.size()); for (Partition partition : partitions) { Map<String, String> partitionSpec = Warehouse.makeSpecFromValues(table.getPartitionKeys(), partition.getValues()); ExprNodeGenericFuncDesc partitionExpression; try { partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); } catch (SemanticException e) { throw new RuntimeException("Unable to build expression", e); } ObjectPair<Integer, byte[]> serializedPartitionExpression = new ObjectPair<>(partitionSpec.size(), SerializationUtilities.serializeExpressionToKryo(partitionExpression)); pairs.add(serializedPartitionExpression); } return pairs; }
private void dropPartitionsUsingExpressions(Table table, Map<String, String> partitionSpec, boolean ifExists, boolean deleteData) throws SemanticException, TException { LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions."); ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); ObjectPair<Integer, byte[]> serializedPartitionExpression = new ObjectPair<>(partitionSpec.size(), SerializationUtilities.serializeExpressionToKryo(partitionExpression)); hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression), deleteData && !isExternal(table), // Delete data? ifExists, // Fail if table doesn't exist? false); // Need results back? }