Refine search
public String getTableLocation() { return table.getSd().getLocation(); }
public static Path getPath(Table table) { String location = table.getSd().getLocation(); if (location == null) { return null; } return new Path(location); }
final public Path getPath() { String location = tTable.getSd().getLocation(); if (location == null) { return null; } return new Path(location); }
final public Path getPath() { String location = tTable.getSd().getLocation(); if (location == null) { return null; } return new Path(location); }
private Path getQueryWorkingDir(Table table) { return new Path(table.getSd().getLocation(), getQueryId()); }
public Path getDataLocation() { if (table.isPartitioned()) { return new Path(tPartition.getSd().getLocation()); } else { return new Path(table.getTTable().getSd().getLocation()); } }
public Path getDataLocation() { if (table.isPartitioned()) { if (tPartition.getSd() == null) return null; else return new Path(tPartition.getSd().getLocation()); } else { if (table.getTTable() == null || table.getTTable().getSd() == null) return null; else return new Path(table.getTTable().getSd().getLocation()); } }
private PartitionHelper newWarehousePartitionHelper() throws MetaException, WorkerException { String location = table.getTable().getSd().getLocation(); Path tablePath = new Path(location); List<FieldSchema> partitionFields = table.getTable().getPartitionKeys(); List<String> partitionColumns = new ArrayList<>(partitionFields.size()); for (FieldSchema field : partitionFields) { partitionColumns.add(field.getName()); } return new WarehousePartitionHelper(configuration, tablePath, partitionColumns); }
private static List<String> listAllDataPaths(HiveMetastore metastore, String schemaName, String tableName) { ImmutableList.Builder<String> locations = ImmutableList.builder(); Table table = metastore.getTable(schemaName, tableName).get(); if (table.getSd().getLocation() != null) { // For unpartitioned table, there should be nothing directly under this directory. // But including this location in the set makes the directory content assert more // extensive, which is desirable. locations.add(table.getSd().getLocation()); } Optional<List<String>> partitionNames = metastore.getPartitionNames(schemaName, tableName); if (partitionNames.isPresent()) { metastore.getPartitionsByNames(schemaName, tableName, partitionNames.get()).stream() .map(partition -> partition.getSd().getLocation()) .filter(location -> !location.startsWith(table.getSd().getLocation())) .forEach(locations::add); } return locations.build(); }
private static String getPartitionPath(Table table, List<String> partValues) { return partValues.get(1).equalsIgnoreCase("isLocatedOutsideTablePath")? // i.e. Is the partition outside the table-dir? table.getSd().getLocation().replace(table.getTableName(), "location_outside_" + table.getTableName()) + "_" + partValues.get(0) + "_" + partValues.get(1) : null ; // Use defaults... Partitions are put in the table directory. }
private PartitionHelper newMetaStorePartitionHelper() throws MetaException, WorkerException { String user = authenticatedUser == null ? System.getProperty("user.name") : authenticatedUser.getShortUserName(); boolean secureMode = authenticatedUser == null ? false : authenticatedUser.hasKerberosCredentials(); try { IMetaStoreClient metaStoreClient = new UgiMetaStoreClientFactory(metaStoreUri, configuration, authenticatedUser, user, secureMode).newInstance(HCatUtil.getHiveMetastoreClient(configuration)); String tableLocation = table.getTable().getSd().getLocation(); Path tablePath = new Path(tableLocation); return new MetaStorePartitionHelper(metaStoreClient, table.getDatabaseName(), table.getTableName(), tablePath); } catch (IOException e) { throw new WorkerException("Could not create meta store client.", e); } }
@Test(expected = MetaException.class) public void testAddPartitionSpecChangeRootPathDiffInSd() throws Exception { Table table = createTable(); String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; String rootPath1 = table.getSd().getLocation() + "/addPartSdPath/"; String rootPath2 = table.getSd().getLocation() + "/someotherpath/"; Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath1 + "part2007/"); PartitionSpecProxy partitionSpecProxy = buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath, Lists.newArrayList(partition)); partitionSpecProxy.setRootLocation(rootPath2); client.add_partitions_pspec(partitionSpecProxy); }
private Path getPartitionLocation() throws NoSuchObjectException, MetaException, TException { Path partitionLocacation; if (partition.isEmpty()) { partitionLocacation = new Path(table.getSd().getLocation()); } else { // TODO: calculate this instead. Just because we're writing to the location doesn't mean that it'll // always be wanted in the meta store right away. List<Partition> partitionEntries = metaStoreClient.listPartitions(table.getDbName(), table.getTableName(), partition, (short) 1); partitionLocacation = new Path(partitionEntries.get(0).getSd().getLocation()); } return partitionLocacation; }
@Before public void injectMocks() throws Exception { when(mockClient.getTable(DATABASE_NAME, TABLE_NAME)).thenReturn(mockTable); when(mockTable.getDbName()).thenReturn(DATABASE_NAME); when(mockTable.getTableName()).thenReturn(TABLE_NAME); when(mockTable.getPartitionKeys()).thenReturn(PARTITION_KEYS); when(mockTable.getSd()).thenReturn(tableStorageDescriptor); tableStorageDescriptor.setLocation(TABLE_LOCATION); when(mockClient.getPartition(DATABASE_NAME, TABLE_NAME, PARTITIONED_VALUES)).thenReturn(mockPartition); when(mockPartition.getSd()).thenReturn(mockPartitionStorageDescriptor); when(mockPartitionStorageDescriptor.getLocation()).thenReturn(PARTITION_LOCATION); helper = new MetaStorePartitionHelper(mockClient, DATABASE_NAME, TABLE_NAME, TABLE_PATH); }
@Test(expected = MetaException.class) public void testAddPartitionSpecChangeRootPathFromNull() throws Exception { Table table = createTable(); String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); PartitionSpecProxy partitionSpecProxy = buildPartitionSpec(DB_NAME, TABLE_NAME, null, Lists.newArrayList(partition)); partitionSpecProxy.setRootLocation(rootPath1); client.add_partitions_pspec(partitionSpecProxy); }
@Test public void testAddPartitionSpecSetRootPath() throws Exception { Table table = createTable(); String rootPath = table.getSd().getLocation() + "/addPartSpecRootPath/"; String rootPath1 = table.getSd().getLocation() + "/someotherpath/"; Partition partition = buildPartition(DB_NAME, TABLE_NAME, "2007", rootPath + "part2007/"); PartitionSpecProxy partitionSpecProxy = buildPartitionSpec(DB_NAME, TABLE_NAME, rootPath1, Lists.newArrayList(partition)); client.add_partitions_pspec(partitionSpecProxy); Partition resultPart = client.getPartition(DB_NAME, TABLE_NAME, Lists.newArrayList("2007")); Assert.assertEquals(rootPath + "part2007", resultPart.getSd().getLocation()); }
@Test public void testDropTableExternalWithoutPurge() throws Exception { Table table = externalTable; client.dropTable(table.getDbName(), table.getTableName(), true, true, false); Assert.assertTrue("Table path should not be removed", metaStore.isPathExists(new Path(table.getSd().getLocation()))); Assert.assertFalse("Table path should be in trash", metaStore.isPathExistsInTrash(new Path(table.getSd().getLocation()))); }
@Test public void testDropTableWithPurge() throws Exception { Table table = testTables[0]; client.dropTable(table.getDbName(), table.getTableName(), true, true, true); Assert.assertFalse("Table path should be removed", metaStore.isPathExists(new Path(table.getSd().getLocation()))); Assert.assertFalse("Table path should not be in trash", metaStore.isPathExistsInTrash(new Path(table.getSd().getLocation()))); }
@Test public void testDropTableExternalWithPurge() throws Exception { Table table = externalTable; client.dropTable(table.getDbName(), table.getTableName(), true, true, true); Assert.assertTrue("Table path should not be removed", metaStore.isPathExists(new Path(table.getSd().getLocation()))); Assert.assertFalse("Table path should not be in trash", metaStore.isPathExistsInTrash(new Path(table.getSd().getLocation()))); }
@Test public void testDropTableWithoutPurge() throws Exception { Table table = testTables[0]; client.dropTable(table.getDbName(), table.getTableName(), true, true, false); Assert.assertFalse("Table path should be removed", metaStore.isPathExists(new Path(table.getSd().getLocation()))); Assert.assertTrue("Table path should be in trash", metaStore.isPathExistsInTrash(new Path(table.getSd().getLocation()))); }