Refine search
public String getTableLocation() { return table.getSd().getLocation(); }
public static Path getPath(Table table) { String location = table.getSd().getLocation(); if (location == null) { return null; } return new Path(location); }
public void init(HiveConf conf, LogHelper console) throws IOException { this.console = console; dir = new Path(partish.getPartSd().getLocation()); fs = dir.getFileSystem(conf); }
final public Path getPath() { String location = tTable.getSd().getLocation(); if (location == null) { return null; } return new Path(location); }
private static void getCompactionCommands(Table t, List<Partition> partitionList, Hive db, boolean execute, List<String> compactionCommands, CompactionMetaInfo compactionMetaInfo, HiveConf conf, ValidTxnList txns) throws IOException, TException, HiveException { for (Partition p : partitionList) { if (needsCompaction(new Path(p.getSd().getLocation()), conf, compactionMetaInfo, txns)) { compactionCommands.add(getCompactionCommand(t, p)); if (execute) { scheduleCompaction(t, p, db, compactionMetaInfo); } } } } private static void scheduleCompaction(Table t, Partition p, Hive db,
@Override public long getUpdateTime(Table table) throws UpdateNotFoundException { return parseDateForLocation(table.getTTable().getSd().getLocation()); }
final public Path getPath() { String location = tTable.getSd().getLocation(); if (location == null) { return null; } return new Path(location); }
public static String makePartLocation(HCatTable table, Map<String, String> partitionSpec) throws MetaException { return (new Path(table.getSd().getLocation(), Warehouse.makePartPath(partitionSpec))).toUri().toString(); }
private static List<String> listAllDataPaths(HiveMetastore metastore, String schemaName, String tableName) { ImmutableList.Builder<String> locations = ImmutableList.builder(); Table table = metastore.getTable(schemaName, tableName).get(); if (table.getSd().getLocation() != null) { // For unpartitioned table, there should be nothing directly under this directory. // But including this location in the set makes the directory content assert more // extensive, which is desirable. locations.add(table.getSd().getLocation()); } Optional<List<String>> partitionNames = metastore.getPartitionNames(schemaName, tableName); if (partitionNames.isPresent()) { metastore.getPartitionsByNames(schemaName, tableName, partitionNames.get()).stream() .map(partition -> partition.getSd().getLocation()) .filter(location -> !location.startsWith(table.getSd().getLocation())) .forEach(locations::add); } return locations.build(); }
private Path getQueryWorkingDir(Table table) { return new Path(table.getSd().getLocation(), getQueryId()); }
public void collectFileStatus(Warehouse wh, HiveConf conf) throws MetaException, IOException { if (!partish.isTransactionalTable()) { partfileStatus = wh.getFileStatusesForSD(partish.getPartSd()); } else { Path path = new Path(partish.getPartSd().getLocation()); partfileStatus = AcidUtils.getAcidFilesForStats(partish.getTable(), path, conf, null); isMissingAcidState = true; } }
boolean shouldModifyTableLocation(Database dbObj, Table tableObj) throws IOException, MetaException { // Should only be managed tables passed in here. // Check if table is in the default table location based on the old warehouse root. // If so then change the table location to the default based on the current warehouse root. // The existing table directory will also be moved to the new default database directory. String tableLocation = tableObj.getSd().getLocation(); Path oldDefaultTableLocation = oldWh.getDefaultTablePath(dbObj, tableObj.getTableName()); if (arePathsEqual(conf, tableLocation, oldDefaultTableLocation.toString())) { if (hasEquivalentEncryption(encryptionShim, oldDefaultTableLocation, curWhRootPath)) { return true; } else { LOG.info("{} and {} are on different encryption zones. Will not change table location for {}", oldDefaultTableLocation, curWhRootPath, getQualifiedName(tableObj)); } } return false; }
public Path getDataLocation() { if (table.isPartitioned()) { return new Path(tPartition.getSd().getLocation()); } else { return new Path(table.getTTable().getSd().getLocation()); } }
Partition toHivePartition() throws HCatException { Partition hivePtn = new Partition(); hivePtn.setDbName(dbName); hivePtn.setTableName(tableName); hivePtn.setValues(values); hivePtn.setParameters(parameters); if (sd.getLocation() == null) { LOG.warn("Partition location is not set! Attempting to construct default partition location."); try { String partName = Warehouse.makePartName(HCatSchemaUtils.getFieldSchemas(hcatTable.getPartCols()), values); sd.setLocation(new Path(hcatTable.getSd().getLocation(), partName).toString()); } catch(MetaException exception) { throw new HCatException("Could not construct default partition-path for " + hcatTable.getDbName() + "." + hcatTable.getTableName() + "[" + values + "]"); } } hivePtn.setSd(sd); hivePtn.setCreateTime((int) (System.currentTimeMillis() / 1000)); hivePtn.setLastAccessTimeIsSet(false); return hivePtn; }
private static String getPartitionPath(Table table, List<String> partValues) { return partValues.get(1).equalsIgnoreCase("isLocatedOutsideTablePath")? // i.e. Is the partition outside the table-dir? table.getSd().getLocation().replace(table.getTableName(), "location_outside_" + table.getTableName()) + "_" + partValues.get(0) + "_" + partValues.get(1) : null ; // Use defaults... Partitions are put in the table directory. }
private List<Path> getLocationsForTruncate(final RawStore ms, final String catName, final String dbName, final String tableName, final Table table, final List<String> partNames) throws Exception { List<Path> locations = new ArrayList<>(); if (partNames == null) { if (0 != table.getPartitionKeysSize()) { for (Partition partition : ms.getPartitions(catName, dbName, tableName, Integer.MAX_VALUE)) { locations.add(new Path(partition.getSd().getLocation())); } } else { locations.add(new Path(table.getSd().getLocation())); } } else { for (Partition partition : ms.getPartitionsByNames(catName, dbName, tableName, partNames)) { locations.add(new Path(partition.getSd().getLocation())); } } return locations; }
@Test public void testDropPartitionDeleteParentDir() throws Exception { client.dropPartition(DB_NAME, TABLE_NAME, PARTITIONS[0].getValues(), true); client.dropPartition(DB_NAME, TABLE_NAME, PARTITIONS[1].getValues(), true); List<Partition> droppedPartitions = Lists.newArrayList(PARTITIONS[0], PARTITIONS[1]); List<Partition> remainingPartitions = Lists.newArrayList(PARTITIONS[2]); checkPartitionsAfterDelete(TABLE_NAME, droppedPartitions, remainingPartitions, true, false); Path parentPath = new Path(PARTITIONS[0].getSd().getLocation()).getParent(); Assert.assertFalse("The parent path '" + parentPath.toString() + "' should not exist.", metaStore.isPathExists(parentPath)); }
@Test(expected = MetaException.class) public void testAddPartitionSpecChangeRootPathDiffInSd() throws Exception { Table table = createTable(); String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; String rootPath1 = table.getSd().getLocation() + "/addPartSdPath/"; String rootPath2 = table.getSd().getLocation() + "/someotherpath/"; Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath1 + "part2007/"); PartitionSpecProxy partitionSpecProxy = buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath, Lists.newArrayList(partition)); partitionSpecProxy.setRootLocation(rootPath2); client.add_partitions_pspec(partitionSpecProxy); }
public Path getDataLocation() { if (table.isPartitioned()) { if (tPartition.getSd() == null) return null; else return new Path(tPartition.getSd().getLocation()); } else { if (table.getTTable() == null || table.getTTable().getSd() == null) return null; else return new Path(table.getTTable().getSd().getLocation()); } }
@Test public void testAddPartitionsNullLocationInTableToo() throws Exception { createTable(DB_NAME, TABLE_NAME, null); List<Partition> partitions = new ArrayList<>(); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null); partitions.add(partition); client.add_partitions(partitions); Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=2017"); Assert.assertEquals( metaStore.getWarehouseRoot() + "/test_partition_db.db/test_partition_table/year=2017", part.getSd().getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(part.getSd().getLocation()))); }