/** * Utility method that returns a table if one corresponding to the destination * tblDesc is found. Returns null if no such table is found. */ public static Table tableIfExists(ImportTableDesc tblDesc, Hive db) throws HiveException { try { return db.getTable(tblDesc.getDatabaseName(),tblDesc.getTableName()); } catch (InvalidTableException e) { return null; } }
/** * Utility method that returns a table if one corresponding to the destination * tblDesc is found. Returns null if no such table is found. */ private static Table tableIfExists(ImportTableDesc tblDesc, Hive db) throws HiveException { try { return db.getTable(tblDesc.getDatabaseName(),tblDesc.getTableName()); } catch (InvalidTableException e) { return null; } }
public static Task<?> getTableCheckpointTask(ImportTableDesc tableDesc, HashMap<String, String> partSpec, String dumpRoot, HiveConf conf) throws SemanticException { HashMap<String, String> mapProp = new HashMap<>(); mapProp.put(REPL_CHECKPOINT_KEY, dumpRoot); AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableDesc.AlterTableTypes.ADDPROPS); alterTblDesc.setProps(mapProp); alterTblDesc.setOldName( StatsUtils.getFullyQualifiedTableName(tableDesc.getDatabaseName(), tableDesc.getTableName())); if (partSpec != null) { alterTblDesc.setPartSpec(partSpec); } return TaskFactory.get(new DDLWork(new HashSet<>(), new HashSet<>(), alterTblDesc), conf); }
private TaskTracker forNewTable() throws Exception { Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); // If table doesn't exist, allow creating a new one only if the database state is older than the update. // This in-turn applicable for partitions creation as well. if ((parentDb != null) && (!event.replicationSpec().allowReplacementInto(parentDb.getParameters()))) { return tracker; } Iterator<AddPartitionDesc> iterator = event.partitionDescriptions(tableDesc).iterator(); while (iterator.hasNext() && tracker.canAddMoreTasks()) { AddPartitionDesc currentPartitionDesc = iterator.next(); /* the currentPartitionDesc cannot be inlined as we need the hasNext() to be evaluated post the current retrieved lastReplicatedPartition */ addPartition(iterator.hasNext(), currentPartitionDesc, null); } return tracker; }
private static Table createNewTableMetadataObject(ImportTableDesc tblDesc, boolean isRepl) throws SemanticException { Table newTable = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); //so that we know the type of table we are creating: acid/MM to match what was exported newTable.setParameters(tblDesc.getTblProps()); if(tblDesc.isExternal() && AcidUtils.isTransactionalTable(newTable)) { if (isRepl) { throw new SemanticException("External tables may not be transactional: " + Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); } else { throw new AssertionError("Internal error: transactional properties not set properly" + tblDesc.getTblProps()); } } return newTable; }
/** * Helper method to set location properly in partSpec */ private static void fixLocationInPartSpec( FileSystem fs, ImportTableDesc tblDesc, Table table, Warehouse wh, ReplicationSpec replicationSpec, AddPartitionDesc.OnePartitionDesc partSpec, EximUtil.SemanticAnalyzerWrapperContext x) throws MetaException, HiveException, IOException { Path tgtPath = null; if (tblDesc.getLocation() == null) { if (table.getDataLocation() != null) { tgtPath = new Path(table.getDataLocation().toString(), Warehouse.makePartPath(partSpec.getPartSpec())); } else { Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName()), Warehouse.makePartPath(partSpec.getPartSpec())); } } else { tgtPath = new Path(tblDesc.getLocation(), Warehouse.makePartPath(partSpec.getPartSpec())); } FileSystem tgtFs = FileSystem.get(tgtPath.toUri(), x.getConf()); checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x); partSpec.setLocation(tgtPath.toString()); }
Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); return new Path( context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName(), tableDesc.isExternal()),
Warehouse.makePartPath(partSpec.getPartSpec())); } else { Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal()),
static TableLocationTuple tableLocation(ImportTableDesc tblDesc, Database parentDb, TableContext tableContext, Context context) throws MetaException, SemanticException { Warehouse wh = context.warehouse; Path defaultTablePath; if (parentDb == null) { defaultTablePath = wh.getDefaultTablePath(tblDesc.getDatabaseName(), tblDesc.getTableName(), tblDesc.isExternal()); } else { defaultTablePath = wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal() ); } // dont use TableType.EXTERNAL_TABLE.equals(tblDesc.tableType()) since this comes in as managed always for tables. if (tblDesc.isExternal()) { if (tblDesc.getLocation() == null) { // this is the use case when the table got converted to external table as part of migration // related rules to be applied to replicated tables across different versions of hive. return new TableLocationTuple(wh.getDnsPath(defaultTablePath).toString(), true); } String currentLocation = new Path(tblDesc.getLocation()).toUri().getPath(); String newLocation = ReplExternalTables.externalTableLocation(context.hiveConf, currentLocation); LOG.debug("external table {} data location is: {}", tblDesc.getTableName(), newLocation); return new TableLocationTuple(newLocation, false); } Path path = tableContext.waitOnPrecursor() ? wh.getDnsPath(defaultTablePath) : wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal()); return new TableLocationTuple(path.toString(), false); }
if (replicationSpec.isTransactionalTableDump()) { List<String> partNames = isPartitioned(tblDesc) ? event.partitions(tblDesc) : null; ReplTxnWork replTxnWork = new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), partNames, replicationSpec.getValidWriteIdList(), ReplTxnWork.OperationType.REPL_WRITEID_STATE); Task<?> replTxnTask = TaskFactory.get(replTxnWork, context.hiveConf); AcidUtils.getFullTableName(tblDesc.getDatabaseName(), tblDesc.getTableName()), new long[0], new BitSet(), ReplUtils.REPL_BOOTSTRAP_MIGRATION_BASE_WRITE_ID); ReplTxnWork replTxnWork = new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), null, validWriteIdList.writeToString(), ReplTxnWork.OperationType.REPL_WRITEID_STATE); Task<?> replTxnTask = TaskFactory.get(replTxnWork, context.hiveConf);
table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName());
table = createNewTableMetadataObject(tblDesc, false); Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName());
Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); if (parentDb == null){ if (!waitOnPrecursor){ throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tblDesc.getDatabaseName())); tblDesc.setLocation( wh.getDnsPath(new Path( wh.getDefaultDatabasePath(tblDesc.getDatabaseName()), MetaStoreUtils.encodeTableName(tblDesc.getTableName().toLowerCase()) table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); if (parentDb == null){ if (!waitOnPrecursor){ throw new SemanticException(ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tblDesc.getDatabaseName())); tblDesc.getDatabaseName(), tblDesc.getTableName()); return; tblDesc.getDatabaseName(), tblDesc.getTableName()); return; tblDesc.getDatabaseName(), tblDesc.getTableName(), null); if (replicationSpec.isMigratingToTxnTable()) { x.setOpenTxnTask(TaskFactory.get(new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), ReplTxnWork.OperationType.REPL_MIGRATION_OPEN_TXN), x.getConf())); updatedMetadata.setNeedCommitTxn(true); } else { tblDesc.setLocation( wh.getDnsPath(wh.getDefaultTablePath(tblDesc.getDatabaseName(), tblDesc.getTableName(), tblDesc.isExternal()) ).toString());
Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); if (parentDb == null) { if (!tableContext.waitOnPrecursor()) { throw new SemanticException( ErrorMsg.DATABASE_NOT_EXISTS.getMsg(tableDesc.getDatabaseName()));
writeId = txnMgr.getTableWriteId(tblDesc.getDatabaseName(), tblDesc.getTableName()); stmtId = txnMgr.getStmtIdAndIncrement();
Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); LoadTable.TableLocationTuple tableLocationTuple = LoadTable.tableLocation(tableDesc, parentDb, tableContext, context);
try { AddPartitionDesc partsDesc = new AddPartitionDesc(tblDesc.getDatabaseName(), tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); ColumnStatisticsDesc colStatsDesc = new ColumnStatisticsDesc(colStats.getStatsDesc()); colStatsDesc.setTableName(tblDesc.getTableName()); colStatsDesc.setDbName(tblDesc.getDatabaseName()); partDesc.setColStats(new ColumnStatistics(colStatsDesc, colStats.getStatsObj()));