/** * Utility method that returns a table if one corresponding to the destination * tblDesc is found. Returns null if no such table is found. */ public static Table tableIfExists(ImportTableDesc tblDesc, Hive db) throws HiveException { try { return db.getTable(tblDesc.getDatabaseName(),tblDesc.getTableName()); } catch (InvalidTableException e) { return null; } }
/** * Utility method that returns a table if one corresponding to the destination * tblDesc is found. Returns null if no such table is found. */ private static Table tableIfExists(ImportTableDesc tblDesc, Hive db) throws HiveException { try { return db.getTable(tblDesc.getDatabaseName(),tblDesc.getTableName()); } catch (InvalidTableException e) { return null; } }
private ReplicationState initialReplicationState() throws SemanticException { return new ReplicationState( new PartitionState(tableDesc.getTableName(), lastReplicatedPartition) ); }
public static Task<?> getTableCheckpointTask(ImportTableDesc tableDesc, HashMap<String, String> partSpec, String dumpRoot, HiveConf conf) throws SemanticException { HashMap<String, String> mapProp = new HashMap<>(); mapProp.put(REPL_CHECKPOINT_KEY, dumpRoot); AlterTableDesc alterTblDesc = new AlterTableDesc(AlterTableDesc.AlterTableTypes.ADDPROPS); alterTblDesc.setProps(mapProp); alterTblDesc.setOldName( StatsUtils.getFullyQualifiedTableName(tableDesc.getDatabaseName(), tableDesc.getTableName())); if (partSpec != null) { alterTblDesc.setPartSpec(partSpec); } return TaskFactory.get(new DDLWork(new HashSet<>(), new HashSet<>(), alterTblDesc), conf); }
public static Task<?> getTableReplLogTask(ImportTableDesc tableDesc, ReplLogger replLogger, HiveConf conf) throws SemanticException { TableType tableType = tableDesc.isExternal() ? TableType.EXTERNAL_TABLE : tableDesc.tableType(); ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger, tableDesc.getTableName(), tableType); return TaskFactory.get(replLogWork, conf); }
static TableLocationTuple tableLocation(ImportTableDesc tblDesc, Database parentDb, TableContext tableContext, Context context) throws MetaException, SemanticException { Warehouse wh = context.warehouse; Path defaultTablePath; if (parentDb == null) { defaultTablePath = wh.getDefaultTablePath(tblDesc.getDatabaseName(), tblDesc.getTableName(), tblDesc.isExternal()); } else { defaultTablePath = wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal() ); } // dont use TableType.EXTERNAL_TABLE.equals(tblDesc.tableType()) since this comes in as managed always for tables. if (tblDesc.isExternal()) { if (tblDesc.getLocation() == null) { // this is the use case when the table got converted to external table as part of migration // related rules to be applied to replicated tables across different versions of hive. return new TableLocationTuple(wh.getDnsPath(defaultTablePath).toString(), true); } String currentLocation = new Path(tblDesc.getLocation()).toUri().getPath(); String newLocation = ReplExternalTables.externalTableLocation(context.hiveConf, currentLocation); LOG.debug("external table {} data location is: {}", tblDesc.getTableName(), newLocation); return new TableLocationTuple(newLocation, false); } Path path = tableContext.waitOnPrecursor() ? wh.getDnsPath(defaultTablePath) : wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal()); return new TableLocationTuple(path.toString(), false); }
private static Table createNewTableMetadataObject(ImportTableDesc tblDesc, boolean isRepl) throws SemanticException { Table newTable = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); //so that we know the type of table we are creating: acid/MM to match what was exported newTable.setParameters(tblDesc.getTblProps()); if(tblDesc.isExternal() && AcidUtils.isTransactionalTable(newTable)) { if (isRepl) { throw new SemanticException("External tables may not be transactional: " + Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); } else { throw new AssertionError("Internal error: transactional properties not set properly" + tblDesc.getTblProps()); } } return newTable; }
/** * Helper method to set location properly in partSpec */ private static void fixLocationInPartSpec( FileSystem fs, ImportTableDesc tblDesc, Table table, Warehouse wh, ReplicationSpec replicationSpec, AddPartitionDesc.OnePartitionDesc partSpec, EximUtil.SemanticAnalyzerWrapperContext x) throws MetaException, HiveException, IOException { Path tgtPath = null; if (tblDesc.getLocation() == null) { if (table.getDataLocation() != null) { tgtPath = new Path(table.getDataLocation().toString(), Warehouse.makePartPath(partSpec.getPartSpec())); } else { Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName()), Warehouse.makePartPath(partSpec.getPartSpec())); } } else { tgtPath = new Path(tblDesc.getLocation(), Warehouse.makePartPath(partSpec.getPartSpec())); } FileSystem tgtFs = FileSystem.get(tgtPath.toUri(), x.getConf()); checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x); partSpec.setLocation(tgtPath.toString()); }
Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); return new Path( context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName(), tableDesc.isExternal()), child); } else {
Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal()), Warehouse.makePartPath(partSpec.getPartSpec()));
dbname + "." + tblDesc.getTableName(), Long.valueOf(replicationSpec.get(ReplicationSpec.KEY.EVENT_ID))); if (tblDesc.getTableName() == null) { x.getConf().set("import.destination.table", tblDesc.getTableName()); for (AddPartitionDesc addPartitionDesc : partitionDescs) { addPartitionDesc.setTableName(tblDesc.getTableName()); x.getLOG().debug("table " + tblDesc.getTableName() + " exists: metadata checked"); tableExists = true;
x.getLOG().debug("table " + tblDesc.getTableName() + " does not exist"); tablePath = new Path(tblDesc.getLocation()); } else { tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal());
x.getLOG().debug("table " + tblDesc.getTableName() + " does not exist"); table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tablePath = new Path(tblDesc.getLocation()); } else { tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName());
if (replicationSpec.isTransactionalTableDump()) { List<String> partNames = isPartitioned(tblDesc) ? event.partitions(tblDesc) : null; ReplTxnWork replTxnWork = new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), partNames, replicationSpec.getValidWriteIdList(), ReplTxnWork.OperationType.REPL_WRITEID_STATE); Task<?> replTxnTask = TaskFactory.get(replTxnWork, context.hiveConf); AcidUtils.getFullTableName(tblDesc.getDatabaseName(), tblDesc.getTableName()), new long[0], new BitSet(), ReplUtils.REPL_BOOTSTRAP_MIGRATION_BASE_WRITE_ID); ReplTxnWork replTxnWork = new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), null, validWriteIdList.writeToString(), ReplTxnWork.OperationType.REPL_WRITEID_STATE); Task<?> replTxnTask = TaskFactory.get(replTxnWork, context.hiveConf);
STATIC_LOG.debug("table {} location is {}", tblDesc.getTableName(), parsedLocation); tblDesc.setLocation(parsedLocation); x.getInputs().add(toReadEntity(new Path(parsedLocation), x.getConf())); if (tblDesc.getTableName() == null) { x.getConf().set("import.destination.table", tblDesc.getTableName()); for (AddPartitionDesc addPartitionDesc : partitionDescs) { addPartitionDesc.setTableName(tblDesc.getTableName()); x.getLOG().debug("table " + tblDesc.getTableName() + " exists: metadata checked"); tableExists = true; writeId = txnMgr.getTableWriteId(tblDesc.getDatabaseName(), tblDesc.getTableName()); stmtId = txnMgr.getStmtIdAndIncrement();
tblDesc.setLocation(wh.getDefaultTablePath(parentDb, tblDesc.getTableName()).toString()); } else { tblDesc.setLocation( wh.getDnsPath(new Path( wh.getDefaultDatabasePath(tblDesc.getDatabaseName()), MetaStoreUtils.encodeTableName(tblDesc.getTableName().toLowerCase()) table = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName());
tblDesc.getDatabaseName(), tblDesc.getTableName()); return; tblDesc.getDatabaseName(), tblDesc.getTableName()); return; updatedMetadata.set(replicationSpec.getReplicationState(), tblDesc.getDatabaseName(), tblDesc.getTableName(), null); if (replicationSpec.isMigratingToTxnTable()) { x.setOpenTxnTask(TaskFactory.get(new ReplTxnWork(tblDesc.getDatabaseName(), tblDesc.getTableName(), ReplTxnWork.OperationType.REPL_MIGRATION_OPEN_TXN), x.getConf())); updatedMetadata.setNeedCommitTxn(true); tblDesc.setLocation(wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal()).toString()); } else { tblDesc.setLocation( wh.getDnsPath(wh.getDefaultTablePath(tblDesc.getDatabaseName(), tblDesc.getTableName(), tblDesc.isExternal()) ).toString());
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbName, ImportTableDesc tblDesc, Partition partition, ReplicationSpec replicationSpec, HiveConf conf) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbName, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); if (replicationSpec.isInReplicationScope() && tblDesc.isExternal() && !replicationSpec.isMigratingToExternalTable()) { String newLocation = ReplExternalTables .externalTableLocation(conf, partition.getSd().getLocation()); LOG.debug("partition {} has data location: {}", partition, newLocation); partDesc.setLocation(newLocation); } else { partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); } return partsDesc; }
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbname, ImportTableDesc tblDesc, Partition partition) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbname, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); return partsDesc; }
try { AddPartitionDesc partsDesc = new AddPartitionDesc(tblDesc.getDatabaseName(), tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); ColumnStatistics colStats = partition.getColStats(); ColumnStatisticsDesc colStatsDesc = new ColumnStatisticsDesc(colStats.getStatsDesc()); colStatsDesc.setTableName(tblDesc.getTableName()); colStatsDesc.setDbName(tblDesc.getDatabaseName()); partDesc.setColStats(new ColumnStatistics(colStatsDesc, colStats.getStatsObj()));