mparts = new ArrayList<>(results.size()); for (Object[] row : results) { MPartition mpart = new MPartition(); int i = 0; for (Object val : row) { mparts = new ArrayList<>(results.size()); for (Object row : results) { MPartition mpart = new MPartition(); MetaStoreServerUtils.setNestedProperty(mpart, fieldNames.get(0), row, true); mparts.add(mpart);
MPartition newp = convertToMPart(newPart, table, false); MColumnDescriptor oldCD = null; MStorageDescriptor oldSD = oldp.getSd(); if (oldSD != null) { oldCD = oldSD.getCD(); oldp.setValues(newp.getValues()); oldp.setPartitionName(newp.getPartitionName()); boolean isTxn = TxnUtils.isTransactionalTable(table.getParameters()); if (isTxn && areTxnStatsSupported) { String errorMsg = verifyStatsChangeCtx(oldp.getParameters(), newPart.getParameters(), newPart.getWriteId(), validWriteIds, false); if (errorMsg != null) { oldp.setParameters(newPart.getParameters()); if (!TableType.VIRTUAL_VIEW.name().equals(oldp.getTable().getTableType())) { copyMSD(newp.getSd(), oldp.getSd()); if (newp.getCreateTime() != oldp.getCreateTime()) { oldp.setCreateTime(newp.getCreateTime()); if (newp.getLastAccessTime() != oldp.getLastAccessTime()) { oldp.setLastAccessTime(newp.getLastAccessTime()); StatsSetupConst.setBasicStatsState(oldp.getParameters(), StatsSetupConst.FALSE); } else if (validWriteIds != null && newPart.getWriteId() > 0) { StatsSetupConst.setBasicStatsState(oldp.getParameters(), StatsSetupConst.FALSE);
/** * Return true if the current statistics in the Metastore is valid * for the query of the given "txnId" and "queryValidWriteIdList". * * Note that a statistics entity is valid iff * the stats is written by the current query or * the conjunction of the following two are true: * ~ COLUMN_STATE_ACCURATE(CSA) state is true * ~ Isolation-level (snapshot) compliant with the query * @param part MPartition of the stats entity * @param queryValidWriteIdList valid writeId list of the query * @Precondition "part" should be retrieved from the PARTITIONS table. */ private boolean isCurrentStatsValidForTheQuery(MPartition part, String queryValidWriteIdList, boolean isCompleteStatsWriter) throws MetaException { return isCurrentStatsValidForTheQuery(conf, part.getParameters(), part.getWriteId(), queryValidWriteIdList, isCompleteStatsWriter); }
private Partition convertToPart(String catName, String dbName, String tblName, MPartition mpart) throws MetaException { if (mpart == null) { return null; } Partition p = new Partition(convertList(mpart.getValues()), dbName, tblName, mpart.getCreateTime(), mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd(), false), convertMap(mpart.getParameters())); p.setCatName(catName); p.setWriteId(mpart.getWriteId()); return p; }
openTransaction(); if (part != null) { List<MFieldSchema> schemas = part.getTable().getPartitionKeys(); List<String> colNames = new ArrayList<>(); for (MFieldSchema col: schemas) { colNames.add(col.getName()); String partName = FileUtils.makePartName(colNames, part.getValues()); part.getTable().getDatabase().getCatalogName(), part.getTable().getDatabase().getName(), part.getTable().getTableName(), Lists.newArrayList(partName)); part.getTable().getDatabase().getCatalogName(), part.getTable().getDatabase().getName(), part.getTable().getTableName(), Lists.newArrayList(partName)); if (CollectionUtils.isNotEmpty(partColumnGrants)) { String catName = part.getTable().getDatabase().getCatalogName(); String dbName = part.getTable().getDatabase().getName(); String tableName = part.getTable().getTableName(); deletePartitionColumnStatistics(catName, dbName, tableName, partName, part.getValues(), null); } catch (NoSuchObjectException e) { LOG.info("No column statistics records found to delete"); preDropStorageDescriptor(part.getSd());
public void alterPartition(String dbname, String name, Partition newPart) throws InvalidObjectException, MetaException { boolean success = false; try { openTransaction(); name = name.toLowerCase(); dbname = dbname.toLowerCase(); MPartition oldp = getMPartition(dbname, name, newPart.getValues()); MPartition newp = convertToMPart(newPart); if (oldp == null || newp == null) { throw new InvalidObjectException("partition does not exist."); } oldp.setParameters(newPart.getParameters()); copyMSD(newp.getSd(), oldp.getSd()); if (newp.getCreateTime() != oldp.getCreateTime()) { oldp.setCreateTime(newp.getCreateTime()); } if (newp.getLastAccessTime() != oldp.getLastAccessTime()) { oldp.setLastAccessTime(newp.getLastAccessTime()); } // commit the changes success = commitTransaction(); } finally { if (!success) { rollbackTransaction(); } } }
private List<HiveObjectPrivilege> convertPartCols(List<MPartitionColumnPrivilege> privs) { List<HiveObjectPrivilege> result = new ArrayList<>(); for (MPartitionColumnPrivilege priv : privs) { String pname = priv.getPrincipalName(); String authorizer = priv.getAuthorizer(); PrincipalType ptype = PrincipalType.valueOf(priv.getPrincipalType()); MPartition mpartition = priv.getPartition(); MTable mtable = mpartition.getTable(); MDatabase mdatabase = mtable.getDatabase(); HiveObjectRef objectRef = new HiveObjectRef(HiveObjectType.COLUMN, mdatabase.getName(), mtable.getTableName(), mpartition.getValues(), priv.getColumnName()); objectRef.setCatName(mdatabase.getCatalogName()); PrivilegeGrantInfo grantor = new PrivilegeGrantInfo(priv.getPrivilege(), priv.getCreateTime(), priv.getGrantor(), PrincipalType.valueOf(priv.getGrantorType()), priv.getGrantOption()); result.add(new HiveObjectPrivilege(objectRef, pname, ptype, grantor, authorizer)); } return result; }
@Override public List<Partition> getPartitionsWithAuth(String catName, String dbName, String tblName, short max, String userName, List<String> groupNames) throws MetaException, InvalidObjectException { boolean success = false; QueryWrapper queryWrapper = new QueryWrapper(); try { openTransaction(); List<MPartition> mparts = listMPartitions(catName, dbName, tblName, max, queryWrapper); List<Partition> parts = new ArrayList<>(mparts.size()); if (CollectionUtils.isNotEmpty(mparts)) { for (MPartition mpart : mparts) { MTable mtbl = mpart.getTable(); Partition part = convertToPart(mpart); parts.add(part); if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { String partName = Warehouse.makePartName(this.convertToFieldSchemas(mtbl .getPartitionKeys()), part.getValues()); PrincipalPrivilegeSet partAuth = this.getPartitionPrivilegeSet(catName, dbName, tblName, partName, userName, groupNames); part.setPrivileges(partAuth); } } } success = commitTransaction(); return parts; } finally { rollbackAndCleanup(success, queryWrapper); } }
} else { MPartition mpart = mparts.get(0); if (name.equals(mpart.getPartitionName())) { ret = mpart; } else { throw new MetaException("Expecting a partition with name " + name + ", but metastore is returning a partition with name " + mpart.getPartitionName() + ".");
Map<String, String> newParams = new HashMap<>(mPartition.getParameters()); StatsSetupConst.setColumnStatsState(newParams, colNames); boolean isTxn = TxnUtils.isTransactionalTable(table); } else { String errorMsg = verifyStatsChangeCtx( mPartition.getParameters(), newParams, writeId, validWriteIds, true); if (errorMsg != null) { throw new MetaException(errorMsg); + statsDesc.getDbName() + "." + statsDesc.getTableName() + "." + statsDesc.getPartName()); mPartition.setWriteId(writeId); mPartition.setParameters(newParams); committed = commitTransaction();
oldStats.get(statsObj.getColName())); Map<String, String> parameters = mPartition.getParameters(); StatsSetupConst.setColumnStatsState(parameters, colNames); mPartition.setParameters(parameters); committed = commitTransaction(); return committed;
MStorageDescriptor mSDS = mPartition.getSd(); List<MFieldSchema> colList = mSDS.getCD().getCols();
LOG.debug("The current metastore transactional partition column statistics for {}.{}.{} " + "(write ID {}) are not valid for current query ({} {})", dbName, tableName, mpart.getPartitionName(), mpart.getWriteId(), writeIdList);
private Partition convertToPart(MPartition mpart) throws MetaException { if (mpart == null) { return null; } //its possible that MPartition is partially filled, do null checks to avoid NPE MTable table = mpart.getTable(); String dbName = table == null ? null : table.getDatabase() == null ? null : table.getDatabase().getName(); String tableName = table == null ? null : table.getTableName(); String catName = table == null ? null : table.getDatabase() == null ? null : table.getDatabase().getCatalogName(); Partition p = new Partition(convertList(mpart.getValues()), dbName, tableName, mpart.getCreateTime(), mpart.getLastAccessTime(), convertToStorageDescriptor(mpart.getSd()), convertMap(mpart.getParameters())); p.setCatName(catName); p.setWriteId(mpart.getWriteId()); return p; }
openTransaction(); if (part != null) { List<MFieldSchema> schemas = part.getTable().getPartitionKeys(); List<String> colNames = new ArrayList<String>(); for (MFieldSchema col: schemas) { colNames.add(col.getName()); String partName = FileUtils.makePartName(colNames, part.getValues()); part.getTable().getDatabase().getName(), part.getTable().getTableName(), Lists.newArrayList(partName)); part.getTable().getDatabase().getName(), part.getTable().getTableName(), Lists.newArrayList(partName)); if (partColumnGrants != null && partColumnGrants.size() > 0) { String dbName = part.getTable().getDatabase().getName(); String tableName = part.getTable().getTableName(); deletePartitionColumnStatistics(dbName, tableName, partName, part.getValues(), null); } catch (NoSuchObjectException e) { LOG.info("No column statistics records found to delete"); preDropStorageDescriptor(part.getSd()); pm.deletePersistent(part);
private List<HiveObjectPrivilege> convertPartition(List<MPartitionPrivilege> privs) { List<HiveObjectPrivilege> result = new ArrayList<>(); for (MPartitionPrivilege priv : privs) { String pname = priv.getPrincipalName(); String authorizer = priv.getAuthorizer(); PrincipalType ptype = PrincipalType.valueOf(priv.getPrincipalType()); MPartition mpartition = priv.getPartition(); MTable mtable = mpartition.getTable(); MDatabase mdatabase = mtable.getDatabase(); HiveObjectRef objectRef = new HiveObjectRef(HiveObjectType.PARTITION, mdatabase.getName(), mtable.getTableName(), mpartition.getValues(), null); objectRef.setCatName(mdatabase.getCatalogName()); PrivilegeGrantInfo grantor = new PrivilegeGrantInfo(priv.getPrivilege(), priv.getCreateTime(), priv.getGrantor(), PrincipalType.valueOf(priv.getGrantorType()), priv.getGrantOption()); result.add(new HiveObjectPrivilege(objectRef, pname, ptype, grantor, authorizer)); } return result; }
MTable mtbl = mpart.getTable(); part = convertToPart(mpart); if ("TRUE".equalsIgnoreCase(mtbl.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
String partName = null; if (partObj != null) { partName = partObj.getPartitionName(); List<MPartitionPrivilege> partPrivs = this .listPrincipalMPartitionGrants(userName, principalType, catName, hiveObject.getDbName(), hiveObject .getObjectName(), partObj.getPartitionName(), authorizer); if (partPrivs != null) { for (MPartitionPrivilege priv : partPrivs) { .getObjectName(), partObj.getPartitionName(), hiveObject.getColumnName(), authorizer); + hiveObject.getDbName() + "," + hiveObject.getObjectName() + "," + partObj.getPartitionName() + "] by " + grantor);
MStorageDescriptor mSDS = mPartition.getSd(); List<MFieldSchema> colList = mSDS.getCD().getCols();
private void alterPartitionNoTxn(String dbname, String name, List<String> part_vals, Partition newPart) throws InvalidObjectException, MetaException { name = name.toLowerCase(); dbname = dbname.toLowerCase(); MPartition oldp = getMPartition(dbname, name, part_vals); MPartition newp = convertToMPart(newPart, false); if (oldp == null || newp == null) { throw new InvalidObjectException("partition does not exist."); } oldp.setValues(newp.getValues()); oldp.setPartitionName(newp.getPartitionName()); oldp.setParameters(newPart.getParameters()); if (!TableType.VIRTUAL_VIEW.name().equals(oldp.getTable().getTableType())) { copyMSD(newp.getSd(), oldp.getSd()); } if (newp.getCreateTime() != oldp.getCreateTime()) { oldp.setCreateTime(newp.getCreateTime()); } if (newp.getLastAccessTime() != oldp.getLastAccessTime()) { oldp.setLastAccessTime(newp.getLastAccessTime()); } }