private static boolean externalTablePartition(ImportTableDesc tblDesc, ReplicationSpec replicationSpec) { return (replicationSpec != null) && replicationSpec.isInReplicationScope() && tblDesc.isExternal(); }
public static Task<?> getTableReplLogTask(ImportTableDesc tableDesc, ReplLogger replLogger, HiveConf conf) throws SemanticException { TableType tableType = tableDesc.isExternal() ? TableType.EXTERNAL_TABLE : tableDesc.tableType(); ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger, tableDesc.getTableName(), tableType); return TaskFactory.get(replLogWork, conf); }
static TableLocationTuple tableLocation(ImportTableDesc tblDesc, Database parentDb, TableContext tableContext, Context context) throws MetaException, SemanticException { Warehouse wh = context.warehouse; Path defaultTablePath; if (parentDb == null) { defaultTablePath = wh.getDefaultTablePath(tblDesc.getDatabaseName(), tblDesc.getTableName(), tblDesc.isExternal()); } else { defaultTablePath = wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal() ); } // dont use TableType.EXTERNAL_TABLE.equals(tblDesc.tableType()) since this comes in as managed always for tables. if (tblDesc.isExternal()) { if (tblDesc.getLocation() == null) { // this is the use case when the table got converted to external table as part of migration // related rules to be applied to replicated tables across different versions of hive. return new TableLocationTuple(wh.getDnsPath(defaultTablePath).toString(), true); } String currentLocation = new Path(tblDesc.getLocation()).toUri().getPath(); String newLocation = ReplExternalTables.externalTableLocation(context.hiveConf, currentLocation); LOG.debug("external table {} data location is: {}", tblDesc.getTableName(), newLocation); return new TableLocationTuple(newLocation, false); } Path path = tableContext.waitOnPrecursor() ? wh.getDnsPath(defaultTablePath) : wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal()); return new TableLocationTuple(path.toString(), false); }
throws MetaException, HiveException { String child = Warehouse.makePartPath(partSpec.getPartSpec()); if (tableDesc.isExternal()) { if (event.replicationSpec().isMigratingToExternalTable()) { return new Path(tableDesc.getLocation(), child); Database parentDb = context.hiveDb.getDatabase(tableDesc.getDatabaseName()); return new Path( context.warehouse.getDefaultTablePath(parentDb, tableDesc.getTableName(), tableDesc.isExternal()), child); } else {
Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal()), Warehouse.makePartPath(partSpec.getPartSpec()));
private static Table createNewTableMetadataObject(ImportTableDesc tblDesc, boolean isRepl) throws SemanticException { Table newTable = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); //so that we know the type of table we are creating: acid/MM to match what was exported newTable.setParameters(tblDesc.getTblProps()); if(tblDesc.isExternal() && AcidUtils.isTransactionalTable(newTable)) { if (isRepl) { throw new SemanticException("External tables may not be transactional: " + Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); } else { throw new AssertionError("Internal error: transactional properties not set properly" + tblDesc.getTblProps()); } } return newTable; }
if (tblDesc.isExternal() && (tblDesc.getLocation() == null)) { x.getLOG().debug("Importing in place, no emptiness check, no copying/loading"); Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME); tablePath = new Path(tblDesc.getLocation()); } else { tablePath = wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal());
if (tblDesc.isExternal() && (tblDesc.getLocation() == null)) { x.getLOG().debug("Importing in place, no emptiness check, no copying/loading"); Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME);
throws MetaException, IOException, HiveException { AddPartitionDesc.OnePartitionDesc partSpec = addPartitionDesc.getPartition(0); if (tblDesc.isExternal() && tblDesc.getLocation() == null) { x.getLOG().debug("Importing in-place: adding AddPart for partition " + partSpecToString(partSpec.getPartSpec()));
if ((tableDesc.isExternal()) // IMPORT statement specified EXTERNAL && (!table.isPartitioned() || !table.getTableType().equals(TableType.EXTERNAL_TABLE)) ){
boolean copyToMigratedTxnTable = false; if (tblDesc.isExternal() && tblDesc.getLocation() == null) { x.getLOG().debug("Importing in-place: adding AddPart for partition " + partSpecToString(partSpec.getPartSpec()));
tblDesc.setLocation(wh.getDefaultTablePath(parentDb, tblDesc.getTableName(), tblDesc.isExternal()).toString()); } else { tblDesc.setLocation( wh.getDnsPath(wh.getDefaultTablePath(tblDesc.getDatabaseName(), tblDesc.getTableName(), tblDesc.isExternal()) ).toString());
if ( (tableDesc.isExternal()) // IMPORT statement speicified EXTERNAL && (!table.isPartitioned() || !table.getTableType().equals(TableType.EXTERNAL_TABLE)) ){
private static AddPartitionDesc getBaseAddPartitionDescFromPartition( Path fromPath, String dbName, ImportTableDesc tblDesc, Partition partition, ReplicationSpec replicationSpec, HiveConf conf) throws MetaException, SemanticException { AddPartitionDesc partsDesc = new AddPartitionDesc(dbName, tblDesc.getTableName(), EximUtil.makePartSpec(tblDesc.getPartCols(), partition.getValues()), partition.getSd().getLocation(), partition.getParameters()); AddPartitionDesc.OnePartitionDesc partDesc = partsDesc.getPartition(0); partDesc.setInputFormat(partition.getSd().getInputFormat()); partDesc.setOutputFormat(partition.getSd().getOutputFormat()); partDesc.setNumBuckets(partition.getSd().getNumBuckets()); partDesc.setCols(partition.getSd().getCols()); partDesc.setSerializationLib(partition.getSd().getSerdeInfo().getSerializationLib()); partDesc.setSerdeParams(partition.getSd().getSerdeInfo().getParameters()); partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); if (replicationSpec.isInReplicationScope() && tblDesc.isExternal() && !replicationSpec.isMigratingToExternalTable()) { String newLocation = ReplExternalTables .externalTableLocation(conf, partition.getSd().getLocation()); LOG.debug("partition {} has data location: {}", partition, newLocation); partDesc.setLocation(newLocation); } else { partDesc.setLocation(new Path(fromPath, Warehouse.makePartName(tblDesc.getPartCols(), partition.getValues())).toString()); } return partsDesc; }
partDesc.setBucketCols(partition.getSd().getBucketCols()); partDesc.setSortCols(partition.getSd().getSortCols()); if (tblDesc.isExternal() && !replicationSpec().isMigratingToExternalTable()) {