/** * Checks if a table is a transactional table that only supports INSERT, but not UPDATE/DELETE * @param params table properties * @return true if table is an INSERT_ONLY table, false otherwise */ public static boolean isInsertOnlyTable(Map<String, String> params) { return isInsertOnlyTable(params, false); } public static boolean isInsertOnlyTable(Table table) {
public static boolean isFullAcidTable(Map<String, String> params) { return isTransactionalTable(params) && !isInsertOnlyTable(params); }
public boolean isMmTable() { return AcidUtils.isInsertOnlyTable(table.getProperties()); }
/** * Should produce the same result as * {@link org.apache.hadoop.hive.metastore.txn.TxnUtils#isAcidTable(org.apache.hadoop.hive.metastore.api.Table)} */ public static boolean isFullAcidTable(org.apache.hadoop.hive.metastore.api.Table table) { return isTransactionalTable(table) && !isInsertOnlyTable(table.getParameters()); }
public boolean isMmTable() { if (getTable() != null) { return AcidUtils.isInsertOnlyTable(table.getParameters()); } else { // Dynamic Partition Insert case return AcidUtils.isInsertOnlyTable(getTableInfo().getProperties()); } } public boolean isFullAcidTable() {
public static boolean isFullAcidTable(CreateTableDesc td) { if (td == null || td.getTblProps() == null) { return false; } String tableIsTransactional = td.getTblProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL); if (tableIsTransactional == null) { tableIsTransactional = td.getTblProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL.toUpperCase()); } return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true") && !AcidUtils.isInsertOnlyTable(td.getTblProps()); }
protected ValidWriteIdList getMmValidWriteIds( JobConf conf, TableDesc table, ValidWriteIdList validWriteIdList) throws IOException { if (!AcidUtils.isInsertOnlyTable(table.getProperties())) return null; if (validWriteIdList == null) { validWriteIdList = AcidUtils.getTableValidWriteIdList( conf, table.getTableName()); if (validWriteIdList == null) { throw new IOException("Insert-Only table: " + table.getTableName() + " is missing from the ValidWriteIdList config: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); } } return validWriteIdList; }
private boolean getNextPath() throws Exception { while (iterPath.hasNext()) { currPath = iterPath.next(); currDesc = iterPartDesc.next(); if (isNonNativeTable) { return true; } FileSystem fs = currPath.getFileSystem(job); if (fs.exists(currPath)) { if (extractValidWriteIdList() != null && AcidUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) { return true; } for (FileStatus fStat : listStatusUnderPath(fs, currPath)) { if (fStat.getLen() > 0) { return true; } } } } return false; }
private void processCurrPathForMmWriteIds(InputFormat inputFormat, List<Path> dirs, List<Path> dirsWithOriginals) throws IOException { if (inputFormat instanceof HiveInputFormat) { dirs.add(currPath); // No need to process here. } ValidWriteIdList validWriteIdList; if (AcidUtils.isInsertOnlyTable(currDesc.getTableDesc().getProperties())) { validWriteIdList = extractValidWriteIdList(); } else { validWriteIdList = null; // non-MM case } if (validWriteIdList != null) { Utilities.FILE_OP_LOGGER.info("Processing " + currDesc.getTableName() + " for MM paths"); } HiveInputFormat.processPathsForMmRead( Lists.newArrayList(currPath), job, validWriteIdList, dirs, dirsWithOriginals); }
private List<Task<?>> alterTableDropProps(AlterTableDesc alterTbl, Table tbl, Partition part, EnvironmentContext environmentContext) throws HiveException { if (StatsSetupConst.USER.equals(environmentContext.getProperties() .get(StatsSetupConst.STATS_GENERATED))) { // drop a stats parameter, which triggers recompute stats update automatically environmentContext.getProperties().remove(StatsSetupConst.DO_NOT_UPDATE_STATS); } List<Task<?>> result = null; if (part == null) { Set<String> removedSet = alterTbl.getProps().keySet(); boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()), isRemoved = AcidUtils.isRemovedInsertOnlyTable(removedSet); if (isFromMmTable && isRemoved) { throw new HiveException("Cannot convert an ACID table to non-ACID"); } // Check if external table property being removed if (removedSet.contains("EXTERNAL") && tbl.getTableType() == TableType.EXTERNAL_TABLE) { tbl.setTableType(TableType.MANAGED_TABLE); } } Iterator<String> keyItr = alterTbl.getProps().keySet().iterator(); while (keyItr.hasNext()) { if (part != null) { part.getTPartition().getParameters().remove(keyItr.next()); } else { tbl.getTTable().getParameters().remove(keyItr.next()); } } return result; }
return null; if (AcidUtils.isInsertOnlyTable(tblproperties)) { LOG.debug("Requesting a major compaction for a MM table; found " + deltas.size() + " delta files, threshold is " + deltaNumThreshold);
public static MmContext createIfNeeded(Table t) { if (t == null) return null; if (!AcidUtils.isInsertOnlyTable(t.getParameters())) return null; return new MmContext(AcidUtils.getFullTableName(t.getDbName(), t.getTableName())); }
public boolean isFullAcidTable() { if(getTable() != null) { return AcidUtils.isFullAcidTable(table); } else { return AcidUtils.isTablePropertyTransactional(getTableInfo().getProperties()) && !AcidUtils.isInsertOnlyTable(getTableInfo().getProperties()); } }
part.getTPartition().getParameters().putAll(alterTbl.getProps()); } else { boolean isFromMmTable = AcidUtils.isInsertOnlyTable(tbl.getParameters()); Boolean isToMmTable = AcidUtils.isToInsertOnlyTable(tbl, alterTbl.getProps()); if (isToMmTable != null) {
if (AcidUtils.isInsertOnlyTable(t.getParameters())) { if (HiveConf.getBoolVar(conf, ConfVars.HIVE_COMPACTOR_COMPACT_MM)) { runMmCompaction(conf, t, p, sd, writeIds, ci);
continue; if (AcidUtils.isInsertOnlyTable(tbl.getParameters())) {
AcidUtils.isInsertOnlyTable(dest_part.getTable().getParameters())) || (table_desc != null && AcidUtils.isInsertOnlyTable(table_desc.getProperties()));
" for " + table.getTableName() + ": " + (AcidUtils.isFullAcidTable(table) ? "acid" : (AcidUtils.isInsertOnlyTable(table) ? "mm" : "flat")
assert tbl.getPath() != null : "null==getPath() for " + tbl.getTableName(); boolean isTxnTable = AcidUtils.isTransactionalTable(tbl); boolean isMmTable = AcidUtils.isInsertOnlyTable(tbl); boolean isFullAcidTable = AcidUtils.isFullAcidTable(tbl);
+ writeId + " for " + partSpecToString(partSpec.getPartSpec()) + ": " + (AcidUtils.isFullAcidTable(table) ? "acid" : (AcidUtils.isInsertOnlyTable(table) ? "mm" : "flat")