private DelegationTokenStore getTokenStore(Configuration conf) throws IOException { String tokenStoreClassName = MetaStoreServerUtils.getTokenStoreClassName(conf); try { Class<? extends DelegationTokenStore> storeClass = Class.forName(tokenStoreClassName).asSubclass(DelegationTokenStore.class); return ReflectionUtils.newInstance(storeClass, conf); } catch (ClassNotFoundException e) { throw new IOException("Error initializing delegation token store: " + tokenStoreClassName, e); } }
|| !containsAllFastStats(newPart.getParameters())) { return true; return !isFastStatsSame(oldPart, newPart);
EnvironmentContext environmentContext, boolean isCreate) throws MetaException { Map<String,String> params = part.getParameters(); if (!forceRecompute && params != null && containsAllFastStats(params)) return false; if (params == null) { params = new HashMap<>(); if (!isCreate && isTransactionalTable(table.getParameters())) { List<FileStatus> fileStatus = wh.getFileStatusesForLocation(part.getLocation()); populateQuickStats(fileStatus, params); LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE)); updateBasicState(environmentContext, params);
public static int startMetaStore(final HadoopThriftAuthBridge bridge, Configuration conf) throws Exception { int port = findFreePort(); startMetaStore(port, bridge, conf); return port; }
partValsList.add(tmpPart.getValues()); if (MetaStoreServerUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { if (MetaStoreServerUtils.isFastStatsSame(oldTmpPart, tmpPart)) { MetaStoreServerUtils.updateBasicState(environmentContext, tmpPart.getParameters()); } else { MetaStoreServerUtils.updatePartitionStatsFast( tmpPart, tbl, wh, false, true, environmentContext, false);
/** Makes shallow copy of a map to avoid DataNucleus mucking with our objects. */ private Map<String, String> convertMap(Map<String, String> dnMap) { return MetaStoreServerUtils.trimMapNulls(dnMap, MetastoreConf.getBoolVar(getConf(), ConfVars.ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS)); }
@Override public List<PartitionSpec> get_partitions_pspec(final String db_name, final String tbl_name, final int max_parts) throws NoSuchObjectException, MetaException { String[] parsedDbName = parseDbName(db_name, conf); String tableName = tbl_name.toLowerCase(); startTableFunction("get_partitions_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName); List<PartitionSpec> partitionSpecs = null; try { Table table = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tableName); // get_partitions will parse out the catalog and db names itself List<Partition> partitions = get_partitions(db_name, tableName, (short) max_parts); if (is_partition_spec_grouping_enabled(table)) { partitionSpecs = MetaStoreServerUtils .getPartitionspecsGroupedByStorageDescriptor(table, partitions); } else { PartitionSpec pSpec = new PartitionSpec(); pSpec.setPartitionList(new PartitionListComposingSpec(partitions)); pSpec.setCatName(parsedDbName[CAT_NAME]); pSpec.setDbName(parsedDbName[DB_NAME]); pSpec.setTableName(tableName); pSpec.setRootPath(table.getSd().getLocation()); partitionSpecs = Arrays.asList(pSpec); } return partitionSpecs; } finally { endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tbl_name); } }
+ " is not a valid object name"); String validate = MetaStoreServerUtils.validateTblColumns(tbl.getSd().getCols()); if (validate != null) { throw new InvalidObjectException("Invalid column " + validate); validate = MetaStoreServerUtils.validateTblColumns(tbl.getPartitionKeys()); if (validate != null) { throw new InvalidObjectException("Invalid partition column " + validate); validate = MetaStoreServerUtils.validateSkewedColNames(skew.getSkewedColNames()); if (validate != null) { throw new InvalidObjectException("Invalid skew column " + validate); validate = MetaStoreServerUtils.validateSkewedColNamesSubsetCol( skew.getSkewedColNames(), tbl.getSd().getCols()); if (validate != null) { MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, madeDir, false, envContext);
throw new InvalidOperationException(newTblName + " is not a valid object name"); String validate = MetaStoreServerUtils.validateTblColumns(newt.getSd().getCols()); if (validate != null) { throw new InvalidOperationException("Invalid column " + validate); if (MetaStoreServerUtils.requireCalStats(null, null, newt, environmentContext) && !isPartitionedTable) { Database db = msdb.getDatabase(catName, newDbName); MetaStoreServerUtils.updateTableStatsSlow(db, newt, wh, false, true, environmentContext); if(!MetaStoreServerUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) { parts = msdb.getPartitions(catName, dbname, name, -1); for (Partition part : parts) {
@Test public void isFastStatsSameWithNullPartitions() { Partition partition = new Partition(); assertFalse(MetaStoreServerUtils.isFastStatsSame(null, null)); assertFalse(MetaStoreServerUtils.isFastStatsSame(null, partition)); assertFalse(MetaStoreServerUtils.isFastStatsSame(partition, null)); }
LOG.warn("Partition/partfiles is null for: " + spec); if (isMissingAcidState) { MetaStoreServerUtils.clearQuickStats(parameters); return p.getOutput(); MetaStoreServerUtils.populateQuickStats(partfileStatus, parameters);
public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { startMetaStore(port, bridge, null); }
/** * Updates the numFiles and totalSize parameters for the passed Partition by querying * the warehouse if the passed Partition does not already have values for these parameters. * @param part * @param wh * @param madeDir if true, the directory was just created and can be assumed to be empty * @param forceRecompute Recompute stats even if the passed Partition already has * these parameters set * @return true if the stats were updated, false otherwise */ public static boolean updatePartitionStatsFast(Partition part, Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext, boolean isCreate) throws MetaException { return updatePartitionStatsFast(new PartitionSpecProxy.SimplePartitionWrapperIterator(part), tbl, wh, madeDir, forceRecompute, environmentContext, isCreate); }
@Test public void testcolumnsIncludedByNameType() { FieldSchema col1 = new FieldSchema("col1", "string", "col1 comment"); FieldSchema col1a = new FieldSchema("col1", "string", "col1 but with a different comment"); FieldSchema col2 = new FieldSchema("col2", "string", "col2 comment"); FieldSchema col3 = new FieldSchema("col3", "string", "col3 comment"); Assert.assertTrue(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1))); Assert.assertTrue(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1), Arrays.asList(col1a))); Assert.assertTrue(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2))); Assert.assertTrue(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col2, col1))); Assert.assertTrue(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1, col2, col3))); Assert.assertTrue(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col3, col2, col1))); Assert.assertFalse(MetaStoreServerUtils.columnsIncludedByNameType(Arrays.asList(col1, col2), Arrays.asList(col1))); }
/** * Verify that the call to updateTableStatsSlow() removes DO_NOT_UPDATE_STATS from table params. */ @Test public void testUpdateTableStatsSlow_removesDoNotUpdateStats() throws TException { // Create database and table Table tbl = new TableBuilder() .setDbName(DB_NAME) .setTableName(TABLE_NAME) .addCol("id", "int") .addTableParam(StatsSetupConst.DO_NOT_UPDATE_STATS, "true") .build(null); Table tbl1 = new TableBuilder() .setDbName(DB_NAME) .setTableName(TABLE_NAME) .addCol("id", "int") .addTableParam(StatsSetupConst.DO_NOT_UPDATE_STATS, "false") .build(null); Warehouse wh = mock(Warehouse.class); MetaStoreServerUtils.updateTableStatsSlow(db, tbl, wh, false, true, null); assertThat(tbl.getParameters(), is(Collections.emptyMap())); verify(wh, never()).getFileStatusesForUnpartitionedTable(db, tbl); MetaStoreServerUtils.updateTableStatsSlow(db, tbl1, wh, true, false, null); assertThat(tbl.getParameters(), is(Collections.emptyMap())); verify(wh, never()).getFileStatusesForUnpartitionedTable(db, tbl1); }
if (!forceRecompute && params != null && containsAllFastStats(params)) { return; populateQuickStats(fileStatus, params); LOG.info("Updated size of table {} to {}", tbl.getTableName(), params.get(StatsSetupConst.TOTAL_SIZE));
part.setValues(part_vals); MetaStoreServerUtils.validatePartitionNameCharacters(part_vals, partitionValidationPattern); MetaStoreServerUtils.updatePartitionStatsFast(part, tbl, wh, madeDir, false, envContext, true);
String newVal = propsFromConf.getProperty(key); if (key.equals(ConfVars.CONNECT_URL_KEY.getVarname())) { oldVal = MetaStoreServerUtils.anonymizeConnectionURL(oldVal); newVal = MetaStoreServerUtils.anonymizeConnectionURL(newVal);
if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { if (MetaStoreServerUtils.isFastStatsSame(oldPart, new_part)) { MetaStoreServerUtils.updateBasicState(environmentContext, new_part.getParameters()); } else { MetaStoreServerUtils.updatePartitionStatsFast( new_part, tbl, wh, false, true, environmentContext, false); if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { MetaStoreServerUtils.updatePartitionStatsFast( new_part, tbl, wh, false, true, environmentContext, false);
t.setParameters(MetaStoreServerUtils.trimMapNulls(t.getParameters(), convertMapNullsToEmptyStrings));