/** * Compute the target location for a Hive location. * @param sourceFs Source {@link FileSystem}. * @param path source {@link Path} in Hive location. * @param partition partition these paths correspond to. * @return transformed location in the target. * @throws IOException if cannot generate a single target location. */ Path getTargetLocation(FileSystem sourceFs, FileSystem targetFs, Path path, Optional<Partition> partition) throws IOException { return getTargetPathHelper().getTargetPath(path, targetFs, partition, false); }
@Test public void testResolvePath() throws Exception { Assert.assertEquals(HiveTargetPathHelper.resolvePath("/data/$DB/$TABLE", "db", "table"), new Path("/data/db/table")); Assert.assertEquals(HiveTargetPathHelper.resolvePath("/data/$TABLE", "db", "table"), new Path("/data/table")); Assert.assertEquals(HiveTargetPathHelper.resolvePath("/data", "db", "table"), new Path("/data/table")); }
Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get());
private HiveTargetPathHelper createTestTargetPathHelper(Properties properties) { HiveDataset dataset = Mockito.mock(HiveDataset.class); Table table = new Table(new org.apache.hadoop.hive.metastore.api.Table()); table.setDbName("dbName"); table.setTableName("tableName"); Mockito.when(dataset.getTable()).thenReturn(table); Mockito.when(dataset.getTableRootPath()).thenReturn(Optional.of(TABLE_ROOT)); Mockito.when(dataset.getProperties()).thenReturn(properties); HiveTargetPathHelper helper = new HiveTargetPathHelper(dataset); return helper; }
this.targetFs = targetFs; this.targetPathHelper = new HiveTargetPathHelper(this.dataset); this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY));
/** * * @param entity, name of the entity to be changed, e.g. hive table or partition * @param sd, StorageDescriptor of the entity */ public static void updateAvroSchemaURL(String entity, StorageDescriptor sd, HiveCopyEntityHelper hiveHelper) { String oldAvroSchemaURL = sd.getSerdeInfo().getParameters().get(HIVE_TABLE_AVRO_SCHEMA_URL); if (oldAvroSchemaURL != null) { Path oldAvroSchemaPath = new Path(oldAvroSchemaURL); URI sourceFileSystemURI = hiveHelper.getDataset().getFs().getUri(); if (PathUtils.isAbsoluteAndSchemeAuthorityNull(oldAvroSchemaPath) || (oldAvroSchemaPath.toUri().getScheme().equals(sourceFileSystemURI.getScheme()) && oldAvroSchemaPath.toUri().getAuthority().equals(sourceFileSystemURI.getAuthority()))) { String newAvroSchemaURL = hiveHelper.getTargetPathHelper().getTargetPath(oldAvroSchemaPath, hiveHelper.getTargetFileSystem(), Optional.<Partition>absent(), true).toString(); sd.getSerdeInfo().getParameters().put(HIVE_TABLE_AVRO_SCHEMA_URL, newAvroSchemaURL); log.info(String.format("For entity %s, change %s from %s to %s", entity, HIVE_TABLE_AVRO_SCHEMA_URL, oldAvroSchemaURL, newAvroSchemaURL)); } } }
this.targetFs = targetFs; this.targetPathHelper = new HiveTargetPathHelper(this.dataset); this.hiveRegProps = new HiveRegProps(new State(this.dataset.getProperties())); this.targetURI = Optional.fromNullable(this.dataset.getProperties().getProperty(TARGET_METASTORE_URI_KEY));
public HiveTargetPathHelper(HiveDataset dataset) { this.dataset = dataset; this.relocateDataFiles = Boolean .valueOf(this.dataset.getProperties().getProperty(RELOCATE_DATA_FILES_KEY, DEFAULT_RELOCATE_DATA_FILES)); this.targetTableRoot = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_ROOT) ? Optional.of(resolvePath(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_ROOT), this.dataset.getTable().getDbName(), this.dataset.getTable().getTableName())) : Optional.<Path> absent(); this.targetTablePrefixTobeReplaced = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED) ? Optional.of(new Path(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED))) : Optional.<Path> absent(); this.targetTablePrefixReplacement = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_PREFIX_REPLACEMENT) ? Optional.of(new Path(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_PREFIX_REPLACEMENT))) : Optional.<Path> absent(); }
Path path = this.targetTableRoot.get(); if (partition.isPresent()) { path = addPartitionToPath(path, partition.get());
@Test public void testRelocateFilesUnpartitioned() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.RELOCATE_DATA_FILES_KEY, Boolean.toString(true)); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/target/tableName/file1")); }
public HiveTargetPathHelper(HiveDataset dataset) { this.dataset = dataset; this.relocateDataFiles = Boolean .valueOf(this.dataset.getProperties().getProperty(RELOCATE_DATA_FILES_KEY, DEFAULT_RELOCATE_DATA_FILES)); this.targetTableRoot = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_ROOT) ? Optional.of(resolvePath(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_ROOT), this.dataset.getTable().getDbName(), this.dataset.getTable().getTableName())) : Optional.<Path> absent(); this.targetTablePrefixTobeReplaced = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED) ? Optional.of(new Path(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED))) : Optional.<Path> absent(); this.targetTablePrefixReplacement = this.dataset.getProperties().containsKey(COPY_TARGET_TABLE_PREFIX_REPLACEMENT) ? Optional.of(new Path(this.dataset.getProperties().getProperty(COPY_TARGET_TABLE_PREFIX_REPLACEMENT))) : Optional.<Path> absent(); }
@Test public void testTokenReplacement() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.RELOCATE_DATA_FILES_KEY, Boolean.toString(true)); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target/$DB/$TABLE"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/target/dbName/tableName/file1")); }
@Test public void testReplacePrefix() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, "/table"); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_PREFIX_REPLACEMENT, "/replaced"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/replaced/path/partition/file1")); }
@Test public void testNewTableRoot() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/target/tableName/partition/file1")); }
@Test public void testReplicatePaths() { Properties properties = new Properties(); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path(TABLE_ROOT, "partition/file1")); }
@Test public void testRelocateFilesPartitioned() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.RELOCATE_DATA_FILES_KEY, Boolean.toString(true)); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Partition partition = Mockito.mock(Partition.class); Mockito.when(partition.getValues()).thenReturn(Lists.newArrayList("part", "123")); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.of(partition), true), new Path("/target/tableName/part/123/file1")); }
dataFiles.add(new SourceAndDestination(status, getTargetPathHelper().getTargetPath(status.getPath(), this.targetFs, partition, true)));
for (FileStatus sourcePath : sourcePaths.values()) { Path newPath = helper.getTargetPathHelper().getTargetPath(sourcePath.getPath(), desiredTargetLocation.getFileSystem(), partition, true); boolean shouldCopy = true; if (desiredTargetExistingPaths.containsKey(newPath)) {
Path targetPath = this.helper.getTargetFileSystem().makeQualified(this.helper.getTargetPathHelper().getTargetPath( input.getPartition().getDataLocation(), this.helper.getTargetFs(), Optional.of(input.getPartition()), false));
HiveTargetPathHelper targetPathHelper = Mockito.mock(HiveTargetPathHelper.class); Mockito.when(targetPathHelper .getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() { @Override