/** * Returns the path object for the given partition key-value pairs and the base location * * @param tblPath - the base location for the partitions. Typically the table location * @param pm - Partition key value pairs * @return * @throws MetaException */ public Path getPartitionPath(Path tblPath, Map<String, String> pm) throws MetaException { return new Path(tblPath, makePartPath(pm)); }
private static Path genPartPathFromTable(Table tbl, Map<String, String> partSpec, Path tblDataLocationPath) throws MetaException { Path partPath = new Path(tbl.getDataLocation(), Warehouse.makePartPath(partSpec)); return new Path(tblDataLocationPath.toUri().getScheme(), tblDataLocationPath.toUri().getAuthority(), partPath.toUri().getPath()); }
private String getAggregationPrefix0(Table table, Partition partition) throws MetaException { // prefix is of the form dbName.tblName String prefix = table.getDbName() + "." + MetaStoreUtils.encodeTableName(table.getTableName()); // FIXME: this is a secret contract; reusein getAggrKey() creates a more closer relation to the StatsGatherer // prefix = work.getAggKey(); prefix = prefix.toLowerCase(); if (partition != null) { return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); } return prefix; }
public static String makePartLocation(HCatTable table, Map<String, String> partitionSpec) throws MetaException { return (new Path(table.getSd().getLocation(), Warehouse.makePartPath(partitionSpec))).toUri().toString(); }
/** * Helper method to set location properly in partSpec */ private static void fixLocationInPartSpec( FileSystem fs, ImportTableDesc tblDesc, Table table, Warehouse wh, ReplicationSpec replicationSpec, AddPartitionDesc.OnePartitionDesc partSpec, EximUtil.SemanticAnalyzerWrapperContext x) throws MetaException, HiveException, IOException { Path tgtPath = null; if (tblDesc.getLocation() == null) { if (table.getDataLocation() != null) { tgtPath = new Path(table.getDataLocation().toString(), Warehouse.makePartPath(partSpec.getPartSpec())); } else { Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName()), Warehouse.makePartPath(partSpec.getPartSpec())); } } else { tgtPath = new Path(tblDesc.getLocation(), Warehouse.makePartPath(partSpec.getPartSpec())); } FileSystem tgtFs = FileSystem.get(tgtPath.toUri(), x.getConf()); checkTargetLocationEmpty(tgtFs, tgtPath, replicationSpec, x); partSpec.setLocation(tgtPath.toString()); }
Warehouse.makePartPath(partSpec.getPartSpec())).toString()); LOG.debug("partition spec {} has location set to {} for a table migrating to external table" + " from managed table", if (table.getDataLocation() != null) { tgtPath = new Path(table.getDataLocation().toString(), Warehouse.makePartPath(partSpec.getPartSpec())); } else { Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); tgtPath = new Path( wh.getDefaultTablePath( parentDb, tblDesc.getTableName(), tblDesc.isExternal()), Warehouse.makePartPath(partSpec.getPartSpec())); Warehouse.makePartPath(partSpec.getPartSpec()));
private String getAggregationPrefix(Table table, Partition partition) throws MetaException { // prefix is of the form dbName.tblName String prefix = table.getDbName() + "." + MetaStoreUtils.encodeTableName(table.getTableName()); if (partition != null) { return Utilities.join(prefix, Warehouse.makePartPath(partition.getSpec())); } return prefix; }
private static void createPartitionIfNotExists(HiveEndPoint ep, IMetaStoreClient msClient, HiveConf conf) throws PartitionCreationFailed { if (ep.partitionVals.isEmpty()) { return; } try { org.apache.hadoop.hive.ql.metadata.Table tableObject = new org.apache.hadoop.hive.ql.metadata.Table(msClient.getTable(ep.database, ep.table)); Map<String, String> partSpec = Warehouse.makeSpecFromValues(tableObject.getPartitionKeys(), ep.partitionVals); AddPartitionDesc addPartitionDesc = new AddPartitionDesc(ep.database, ep.table, true); String partLocation = new Path(tableObject.getDataLocation(), Warehouse.makePartPath(partSpec)).toString(); addPartitionDesc.addPartition(partSpec, partLocation); Partition partition = Hive.convertAddSpecToMetaPartition(tableObject, addPartitionDesc.getPartition(0), conf); msClient.add_partition(partition); } catch (AlreadyExistsException e) { //ignore this - multiple clients may be trying to create the same partition //AddPartitionDesc has ifExists flag but it's not propagated to // HMSHnalder.add_partitions_core() and so it throws... } catch(HiveException|TException e) { LOG.error("Failed to create partition : " + ep, e); throw new PartitionCreationFailed(ep, e); } }
public static List<Path> getInputPathsForPartialScan(TableScanOperator tableScanOp, Appendable aggregationKey) throws SemanticException { List<Path> inputPaths = new ArrayList<Path>(); switch (tableScanOp.getConf().getTableMetadata().getTableSpec().specType) { case TABLE_ONLY: inputPaths.add(tableScanOp.getConf().getTableMetadata() .getTableSpec().tableHandle.getPath()); break; case STATIC_PARTITION: Partition part = tableScanOp.getConf().getTableMetadata() .getTableSpec().partHandle; try { aggregationKey.append(Warehouse.makePartPath(part.getSpec())); } catch (MetaException e) { throw new SemanticException(ErrorMsg.ANALYZE_TABLE_PARTIALSCAN_AGGKEY.getMsg( part.getDataLocation().toString() + e.getMessage())); } catch (IOException e) { throw new RuntimeException(e); } inputPaths.add(part.getDataLocation()); break; default: assert false; } return inputPaths; }
String child = Warehouse.makePartPath(partSpec.getPartSpec()); if (tableDesc.isExternal()) { if (event.replicationSpec().isMigratingToExternalTable()) {
@Override public PartitionInfo createPartitionIfNotExists(final List<String> partitionValues) throws StreamingException { String partLocation = null; String partName = null; boolean exists = false; try { Map<String, String> partSpec = Warehouse.makeSpecFromValues(tableObject.getPartitionKeys(), partitionValues); AddPartitionDesc addPartitionDesc = new AddPartitionDesc(database, table, true); partName = Warehouse.makePartName(tableObject.getPartitionKeys(), partitionValues); partLocation = new Path(tableObject.getDataLocation(), Warehouse.makePartPath(partSpec)).toString(); addPartitionDesc.addPartition(partSpec, partLocation); Partition partition = Hive.convertAddSpecToMetaPartition(tableObject, addPartitionDesc.getPartition(0), conf); if (getMSC() == null) { // We assume it doesn't exist if we can't check it // so the driver will decide return new PartitionInfo(partName, partLocation, false); } getMSC().add_partition(partition); if (LOG.isDebugEnabled()) { LOG.debug("Created partition {} for table {}", partName, tableObject.getFullyQualifiedName()); } } catch (AlreadyExistsException e) { exists = true; } catch (HiveException | TException e) { throw new StreamingException("Unable to creation partition for values: " + partitionValues + " connection: " + toConnectionInfoString(), e); } return new PartitionInfo(partName, partLocation, exists); }
PartitionResult pr = new PartitionResult(); pr.setTableName(tableName); pr.setPartitionName(Warehouse.makePartPath(map)); result.getPartitionsNotInMs().add(pr); } else {
CheckResult.PartitionResult pr = new CheckResult.PartitionResult(); pr.setTableName(tableName); pr.setPartitionName(Warehouse.makePartPath(map)); result.getPartitionsNotInMs().add(pr); } else {
Warehouse.makePartPath(partSpec)); newPartPath = new Path(tblDataLocationPath.toUri().getScheme(), tblDataLocationPath.toUri().getAuthority(), partPath.toUri().getPath());
table.getPartitionKeys(), partitionValues); try { destLocation = new Path(table.getDataLocation(), Warehouse.makePartPath(partSpec)); } catch (MetaException e) { throw new StreamingException("Unable to retrieve the delta file location"
String staticSpec = Warehouse.makePartPath(dest_part.getSpec()); fileSinkDesc.setStaticSpec(staticSpec); } catch (MetaException e) {
try { desc.setLocation(new Path(tab.getDataLocation(), Warehouse.makePartPath(desc.getPartSpec())).toString());
String staticSpec = Warehouse.makePartPath(dest_part.getSpec()); fileSinkDesc.setStaticSpec(staticSpec); } catch (MetaException e) {
/** * Returns the path object for the given partition key-value pairs and the base location * * @param tblPath - the base location for the partitions. Typically the table location * @param pm - Partition key value pairs * @return * @throws MetaException */ public Path getPartitionPath(Path tblPath, Map<String, String> pm) throws MetaException { return new Path(tblPath, makePartPath(pm)); }
public Path getPartitionPath(Database db, String tableName, LinkedHashMap<String, String> pm) throws MetaException { return new Path(getTablePath(db, tableName), makePartPath(pm)); }