private void doDistCpCopyOnce(FileSystem sourceFs, List<Path> srcList, Path destination, boolean usePrivilegedUser) throws IOException { if (hiveConf.getBoolVar(HiveConf.ConfVars.REPL_ADD_RAW_RESERVED_NAMESPACE)) { srcList = srcList.stream().map(path -> { URI uri = path.toUri(); return new Path(uri.getScheme(), uri.getAuthority(), RAW_RESERVED_VIRTUAL_PATH + uri.getPath()); }).collect(Collectors.toList()); URI destinationUri = destination.toUri(); destination = new Path(destinationUri.getScheme(), destinationUri.getAuthority(), RAW_RESERVED_VIRTUAL_PATH + destinationUri.getPath()); } if (!FileUtils.distCp( sourceFs, // source file system srcList, // list of source paths destination, false, usePrivilegedUser ? copyAsUser : null, hiveConf, ShimLoader.getHadoopShims())) { LOG.error("Distcp failed to copy files: " + srcList + " to destination: " + destination); throw new IOException("Distcp operation failed."); } }
FileUtils.distCp(
LOG.info("Launch distributed copy (distcp) job."); triedDistcp = true; copied = distCp(srcFS, Collections.singletonList(src), dst, deleteSource, null, conf, shims);
Assert.assertTrue(FileUtils.distCp(fs, Collections.singletonList(copySrc), copyDst, false, doAsUser, conf, shims)); verify(shims).runDistCpAs(Collections.singletonList(copySrc), copyDst, conf, doAsUser); Assert.assertFalse(FileUtils.distCp(fs, Collections.singletonList(copySrc), copyDst, true, null, conf, shims)); verify(shims).runDistCp(Collections.singletonList(copySrc), copyDst, conf); FileUtils.distCp(fs, Collections.singletonList(copySrc), copyDst, true, doAsUser, conf, shims); Assert.assertTrue("Should throw IOException as doAs is called with delete source set to true".equals("")); } catch (IOException e) {
@Test(expected = IOException.class) public void shouldThrowExceptionOnDistcpFailure() throws Exception { Path destination = mock(Path.class); Path source = mock(Path.class); FileSystem fs = mock(FileSystem.class); List<Path> srcPaths = Arrays.asList(source, source); HiveConf conf = mock(HiveConf.class); CopyUtils copyUtils = Mockito.spy(new CopyUtils(null, conf)); mockStatic(FileUtils.class); mockStatic(Utils.class); when(destination.getFileSystem(same(conf))).thenReturn(fs); when(source.getFileSystem(same(conf))).thenReturn(fs); when(FileUtils.distCp(same(fs), anyListOf(Path.class), same(destination), anyBoolean(), eq(null), same(conf), same(ShimLoader.getHadoopShims()))) .thenReturn(false); when(Utils.getUGI()).thenReturn(mock(UserGroupInformation.class)); doReturn(false).when(copyUtils).regularCopy(same(fs), same(fs), anyListOf(ReplChangeManager.FileInfo.class)); copyUtils.doCopy(destination, srcPaths); } }
LOG.info("Launch distributed copy (distcp) job."); triedDistcp = true; copied = distCp(srcFS, Collections.singletonList(src), dst, deleteSource, null, conf, shims);