/** * Compute the target location for a Hive location. * @param sourceFs Source {@link FileSystem}. * @param path source {@link Path} in Hive location. * @param partition partition these paths correspond to. * @return transformed location in the target. * @throws IOException if cannot generate a single target location. */ Path getTargetLocation(FileSystem sourceFs, FileSystem targetFs, Path path, Optional<Partition> partition) throws IOException { return getTargetPathHelper().getTargetPath(path, targetFs, partition, false); }
/** * * @param entity, name of the entity to be changed, e.g. hive table or partition * @param sd, StorageDescriptor of the entity */ public static void updateAvroSchemaURL(String entity, StorageDescriptor sd, HiveCopyEntityHelper hiveHelper) { String oldAvroSchemaURL = sd.getSerdeInfo().getParameters().get(HIVE_TABLE_AVRO_SCHEMA_URL); if (oldAvroSchemaURL != null) { Path oldAvroSchemaPath = new Path(oldAvroSchemaURL); URI sourceFileSystemURI = hiveHelper.getDataset().getFs().getUri(); if (PathUtils.isAbsoluteAndSchemeAuthorityNull(oldAvroSchemaPath) || (oldAvroSchemaPath.toUri().getScheme().equals(sourceFileSystemURI.getScheme()) && oldAvroSchemaPath.toUri().getAuthority().equals(sourceFileSystemURI.getAuthority()))) { String newAvroSchemaURL = hiveHelper.getTargetPathHelper().getTargetPath(oldAvroSchemaPath, hiveHelper.getTargetFileSystem(), Optional.<Partition>absent(), true).toString(); sd.getSerdeInfo().getParameters().put(HIVE_TABLE_AVRO_SCHEMA_URL, newAvroSchemaURL); log.info(String.format("For entity %s, change %s from %s to %s", entity, HIVE_TABLE_AVRO_SCHEMA_URL, oldAvroSchemaURL, newAvroSchemaURL)); } } }
@Test public void testRelocateFilesUnpartitioned() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.RELOCATE_DATA_FILES_KEY, Boolean.toString(true)); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/target/tableName/file1")); }
@Test public void testTokenReplacement() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.RELOCATE_DATA_FILES_KEY, Boolean.toString(true)); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target/$DB/$TABLE"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/target/dbName/tableName/file1")); }
@Test public void testReplacePrefix() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_PREFIX_TOBE_REPLACED, "/table"); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_PREFIX_REPLACEMENT, "/replaced"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/replaced/path/partition/file1")); }
@Test public void testNewTableRoot() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path("/target/tableName/partition/file1")); }
@Test public void testReplicatePaths() { Properties properties = new Properties(); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.<Partition>absent(), true), new Path(TABLE_ROOT, "partition/file1")); }
@Test public void testRelocateFilesPartitioned() { Properties properties = new Properties(); properties.setProperty(HiveTargetPathHelper.RELOCATE_DATA_FILES_KEY, Boolean.toString(true)); properties.setProperty(HiveTargetPathHelper.COPY_TARGET_TABLE_ROOT, "/target"); HiveTargetPathHelper helper = createTestTargetPathHelper(properties); Path source = new Path(TABLE_ROOT, "partition/file1"); Partition partition = Mockito.mock(Partition.class); Mockito.when(partition.getValues()).thenReturn(Lists.newArrayList("part", "123")); Assert.assertEquals(helper.getTargetPath(source, this.fs, Optional.of(partition), true), new Path("/target/tableName/part/123/file1")); }
dataFiles.add(new SourceAndDestination(status, getTargetPathHelper().getTargetPath(status.getPath(), this.targetFs, partition, true)));
for (FileStatus sourcePath : sourcePaths.values()) { Path newPath = helper.getTargetPathHelper().getTargetPath(sourcePath.getPath(), desiredTargetLocation.getFileSystem(), partition, true); boolean shouldCopy = true; if (desiredTargetExistingPaths.containsKey(newPath)) {
Path targetPath = this.helper.getTargetFileSystem().makeQualified(this.helper.getTargetPathHelper().getTargetPath( input.getPartition().getDataLocation(), this.helper.getTargetFs(), Optional.of(input.getPartition()), false));
HiveTargetPathHelper targetPathHelper = Mockito.mock(HiveTargetPathHelper.class); Mockito.when(targetPathHelper .getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() { @Override
Mockito.when(table.getCompleteName()).thenReturn("table1"); Mockito.when(targetPathHelper .getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() { @Override
Mockito.when(table.getCompleteName()).thenReturn("table1"); Mockito.when(targetPathHelper .getTargetPath(Mockito.any(Path.class), Mockito.any(FileSystem.class), Mockito.any(Optional.class), Mockito.anyBoolean())).then(new Answer<Path>() { @Override
/** * Compute the target location for a Hive location. * @param sourceFs Source {@link FileSystem}. * @param path source {@link Path} in Hive location. * @param partition partition these paths correspond to. * @return transformed location in the target. * @throws IOException if cannot generate a single target location. */ Path getTargetLocation(FileSystem sourceFs, FileSystem targetFs, Path path, Optional<Partition> partition) throws IOException { return getTargetPathHelper().getTargetPath(path, targetFs, partition, false); }
/** * * @param entity, name of the entity to be changed, e.g. hive table or partition * @param sd, StorageDescriptor of the entity */ public static void updateAvroSchemaURL(String entity, StorageDescriptor sd, HiveCopyEntityHelper hiveHelper) { String oldAvroSchemaURL = sd.getSerdeInfo().getParameters().get(HIVE_TABLE_AVRO_SCHEMA_URL); if (oldAvroSchemaURL != null) { Path oldAvroSchemaPath = new Path(oldAvroSchemaURL); URI sourceFileSystemURI = hiveHelper.getDataset().getFs().getUri(); if (PathUtils.isAbsoluteAndSchemeAuthorityNull(oldAvroSchemaPath) || (oldAvroSchemaPath.toUri().getScheme().equals(sourceFileSystemURI.getScheme()) && oldAvroSchemaPath.toUri().getAuthority().equals(sourceFileSystemURI.getAuthority()))) { String newAvroSchemaURL = hiveHelper.getTargetPathHelper().getTargetPath(oldAvroSchemaPath, hiveHelper.getTargetFileSystem(), Optional.<Partition>absent(), true).toString(); sd.getSerdeInfo().getParameters().put(HIVE_TABLE_AVRO_SCHEMA_URL, newAvroSchemaURL); log.info(String.format("For entity %s, change %s from %s to %s", entity, HIVE_TABLE_AVRO_SCHEMA_URL, oldAvroSchemaURL, newAvroSchemaURL)); } } }
dataFiles.add(new SourceAndDestination(status, getTargetPathHelper().getTargetPath(status.getPath(), this.targetFs, partition, true)));
Path targetPath = this.helper.getTargetFileSystem().makeQualified(this.helper.getTargetPathHelper().getTargetPath( input.getPartition().getDataLocation(), this.helper.getTargetFs(), Optional.of(input.getPartition()), false));
for (FileStatus sourcePath : sourcePaths.values()) { Path newPath = helper.getTargetPathHelper().getTargetPath(sourcePath.getPath(), desiredTargetLocation.getFileSystem(), partition, true); boolean shouldCopy = true; if (desiredTargetExistingPaths.containsKey(newPath)) {