/** * Send a heartbeat to the metastore for locks and transactions. */ @Override public void run() { LOG.trace("Heartbeating materialization rebuild lock for {} for query: {}", AcidUtils.getFullTableName(dbName, tableName), queryId); boolean refreshed; try { refreshed = txnMgr.heartbeatMaterializationRebuildLock(dbName, tableName, txnId); } catch (LockException e) { LOG.error("Failed trying to acquire lock", e); throw new RuntimeException(e); } if (!refreshed) { // We could not heartbeat the lock, i.e., the operation has finished, // hence we interrupt this work ScheduledFuture<?> t = task.get(); if (t != null) { t.cancel(false); LOG.debug("Stopped heartbeat for materialization rebuild lock for {} for query: {}", AcidUtils.getFullTableName(dbName, tableName), queryId); } } } }
private String getValidWriteIdList(String dbName, String tblName, String validTxnString) throws LockException { if ((validTxnString == null) || validTxnString.isEmpty()) { return null; } String fullTableName = AcidUtils.getFullTableName(dbName, tblName); ValidWriteIdList validWriteIds = getTxnMgr() .getValidWriteIds(Collections.singletonList(fullTableName), validTxnString) .getTableValidWriteIdList(fullTableName); return ((validWriteIds != null) ? validWriteIds.toString() : null); }
/** * Set the valid write id list for the current table scan. */ public static void setValidWriteIdList(Configuration conf, TableScanDesc tsDesc) { if (tsDesc.isTranscationalTable()) { String dbName = tsDesc.getDatabaseName(); String tableName = tsDesc.getTableName(); ValidWriteIdList validWriteIdList = getTableValidWriteIdList(conf, AcidUtils.getFullTableName(dbName, tableName)); if (validWriteIdList != null) { setValidWriteIdList(conf, validWriteIdList); } else { // Log error if the acid table is missing from the ValidWriteIdList conf LOG.error("setValidWriteIdList on table: " + AcidUtils.getFullTableName(dbName, tableName) + " isAcidTable: " + true + " acidProperty: " + getAcidOperationalProperties(conf) + " couldn't find the ValidWriteId list from ValidTxnWriteIdList: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); throw new IllegalStateException("ACID table: " + AcidUtils.getFullTableName(dbName, tableName) + " is missing from the ValidWriteIdList config: " + conf.get(ValidTxnWriteIdList.VALID_TABLES_WRITEIDS_KEY)); } } }
private long getTableWriteId( String dbName, String tableName, boolean allocateIfNotYet) throws LockException { String fullTableName = AcidUtils.getFullTableName(dbName, tableName); if (tableWriteIds.containsKey(fullTableName)) { return tableWriteIds.get(fullTableName); } else if (!allocateIfNotYet) { return 0; } try { long writeId = getMS().allocateTableWriteId(txnId, dbName, tableName); LOG.debug("Allocated write ID {} for {}.{}", writeId, dbName, tableName); tableWriteIds.put(fullTableName, writeId); return writeId; } catch (TException e) { throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); } }
/** * Returns ValidWriteIdList for the table with the given "dbName" and "tableName". * This is called when HiveConf has no list for the table. * Otherwise use getTableSnapshot(). * @param conf Configuration * @param dbName * @param tableName * @return ValidWriteIdList on success, null on failure to get a list. * @throws LockException */ public static ValidWriteIdList getTableValidWriteIdListWithTxnList( Configuration conf, String dbName, String tableName) throws LockException { HiveTxnManager sessionTxnMgr = SessionState.get().getTxnMgr(); if (sessionTxnMgr == null) { return null; } ValidWriteIdList validWriteIdList = null; ValidTxnWriteIdList validTxnWriteIdList = null; String validTxnList = conf.get(ValidTxnList.VALID_TXNS_KEY); List<String> tablesInput = new ArrayList<>(); String fullTableName = getFullTableName(dbName, tableName); tablesInput.add(fullTableName); validTxnWriteIdList = sessionTxnMgr.getValidWriteIds(tablesInput, validTxnList); return validTxnWriteIdList != null ? validTxnWriteIdList.getTableValidWriteIdList(fullTableName) : null; }
private List<Pair<String, Table>> getWrittenTableList(QueryPlan plan) { List<Pair<String, Table>> result = new ArrayList<>(); Set<String> tableList = new HashSet<>(); for (WriteEntity output : plan.getOutputs()) { Table tbl; switch (output.getType()) { case TABLE: { tbl = output.getTable(); break; } case PARTITION: case DUMMYPARTITION: { tbl = output.getPartition().getTable(); break; } default: { continue; } } String fullTableName = AcidUtils.getFullTableName(tbl.getDbName(), tbl.getTableName()); if (tableList.add(fullTableName)) { result.add(new ImmutablePair(fullTableName, tbl)); } } return result; }
public static MmContext createIfNeeded(Table t) { if (t == null) return null; if (!AcidUtils.isInsertOnlyTable(t.getParameters())) return null; return new MmContext(AcidUtils.getFullTableName(t.getDbName(), t.getTableName())); }
@Override public LockResponse acquireMaterializationRebuildLock(String dbName, String tableName, long txnId) throws LockException { // Acquire lock LockResponse lockResponse; try { lockResponse = getMS().lockMaterializationRebuild(dbName, tableName, txnId); } catch (TException e) { throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e); } if (lockResponse.getState() == LockState.ACQUIRED) { // If lock response is ACQUIRED, we can create the heartbeater long initialDelay = 0L; long heartbeatInterval = getHeartbeatInterval(conf); assert heartbeatInterval > 0; MaterializationRebuildLockHeartbeater heartbeater = new MaterializationRebuildLockHeartbeater( this, dbName, tableName, queryId, txnId); ScheduledFuture<?> task = startHeartbeat(initialDelay, heartbeatInterval, heartbeater); heartbeater.task.set(task); LOG.debug("Started heartbeat for materialization rebuild lock for {} with delay/interval = {}/{} {} for query: {}", AcidUtils.getFullTableName(dbName, tableName), initialDelay, heartbeatInterval, TimeUnit.MILLISECONDS, queryId); } return lockResponse; }
List<FileStatus> fileList = new ArrayList<>(); ValidWriteIdList idList = AcidUtils.getTableValidWriteIdList(jc, AcidUtils.getFullTableName(table.getDbName(), table.getTableName())); if (idList == null) { LOG.warn("Cannot get ACID state for " + table.getDbName() + "." + table.getTableName()
private void addTableFromEntity(Entity entity, Collection<String> tableList) { Table tbl; switch (entity.getType()) { case TABLE: { tbl = entity.getTable(); break; } case PARTITION: case DUMMYPARTITION: { tbl = entity.getPartition().getTable(); break; } default: { return; } } if (!AcidUtils.isTransactionalTable(tbl)) { return; } String fullTableName = AcidUtils.getFullTableName(tbl.getDbName(), tbl.getTableName()); tableList.add(fullTableName); }
/** Creates a new {@link Transaction} by opening a transaction with the {@link IMetaStoreClient meta store}. */ public Transaction newTransaction() throws TransactionException { if (!connected) { throw new TransactionException("Not connected - cannot create transaction."); } Transaction transaction = new Transaction(metaStoreClient, lockOptions); long txnId = transaction.getTransactionId(); for (AcidTable table : tables) { try { table.setWriteId(metaStoreClient.allocateTableWriteId(txnId, table.getDatabaseName(), table.getTableName())); } catch (TException ex) { try { metaStoreClient.rollbackTxn(txnId); } catch (TException e) { LOG.warn("Allocation of write id failed for table {} and rollback transaction {} failed due to {}", AcidUtils.getFullTableName(table.getDatabaseName(), table.getTableName()), txnId, e.getMessage()); } throw new TransactionException("Unable to allocate table write ID for table " + AcidUtils.getFullTableName(table.getDatabaseName(), table.getTableName()) + " under txn " + txnId, ex); } } LOG.debug("Created transaction {}", transaction); return transaction; }
String fullTableName = getFullTableName(dbName, tblName); if (sessionTxnMgr != null && sessionTxnMgr.getCurrentTxnId() > 0) { validWriteIdList = getTableValidWriteIdList(conf, fullTableName);
if (source instanceof TableScanOperator) { TableScanOperator tsOp = (TableScanOperator)source; String fullTableName = AcidUtils.getFullTableName(tsOp.getConf().getDatabaseName(), tsOp.getConf().getTableName()); ValidWriteIdList writeIdList = txnWriteIds.getTableValidWriteIdList(fullTableName);
public int persistColumnStats(Hive db, Table tbl) throws HiveException, MetaException, IOException { // Construct a column statistics object from the result List<ColumnStatistics> colStats = constructColumnStatsFromPackedRows(tbl); // Persist the column statistics object to the metastore // Note, this function is shared for both table and partition column stats. if (colStats.isEmpty()) { return 0; } SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats); request.setNeedMerge(colStatDesc.isNeedMerge()); HiveTxnManager txnMgr = AcidUtils.isTransactionalTable(tbl) ? SessionState.get().getTxnMgr() : null; if (txnMgr != null) { request.setValidWriteIdList(AcidUtils.getTableValidWriteIdList(conf, AcidUtils.getFullTableName(tbl.getDbName(), tbl.getTableName())).toString()); request.setWriteId(txnMgr.getAllocatedTableWriteId(tbl.getDbName(), tbl.getTableName())); } db.setPartitionColumnStatistics(request); return 0; }
AcidUtils.getFullTableName(tblDesc.getDatabaseName(), tblDesc.getTableName()), new long[0], new BitSet(), ReplUtils.REPL_BOOTSTRAP_MIGRATION_BASE_WRITE_ID); ReplTxnWork replTxnWork = new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), null,
snapshot = AcidUtils.getTableSnapshot(conf, table, true); } else { String fullTableName = getFullTableName(table.getDbName(), table.getTableName()); ValidWriteIdList writeIdList = getMSC().getValidWriteIds(fullTableName, writeId); snapshot = new TableSnapshot(writeId, writeIdList.writeToString());