Refine search
public static String makePartName(List<String> partCols, List<String> vals) { return makePartName(partCols, vals, null); }
private String prependPartSpec(String inputPath, String fileName) { Map<String, String> mapping = inputToPartSpecMapping == null ? inputToPartSpecMapping = revert(bigTablePartSpecToFileMapping) : inputToPartSpecMapping; String partSpec = mapping.get(URI.create(inputPath).getPath()); return partSpec == null || partSpec.isEmpty() ? fileName : "(" + FileUtils.escapePathName(partSpec) + ")" + fileName; }
Map<List<String>, String> skewedColValueLocationMaps, Path newPartPath, SkewedInfo skewedInfo) { Path lbdPath = fSta.getPath().getParent(); List<String> skewedValue = new ArrayList<String>(); String lbDirName = FileUtils.unescapePathName(lbdPath.toString()); String partDirName = FileUtils.unescapePathName(newPartPath.toString()); String lbDirSuffix = lbDirName.replace(partDirName, ""); String[] dirNames = lbDirSuffix.split(Path.SEPARATOR);
public static boolean isActionPermittedForFileHierarchy(FileSystem fs, FileStatus fileStatus, String userName, FsAction action, boolean recurse) throws Exception { boolean isDir = fileStatus.isDir(); checkFileAccessWithImpersonation(fs, fileStatus, action, userName, subDirsToCheck); } catch (AccessControlException err) { LOG.warn("Action " + action + " denied on " + fileStatus.getPath() + " for user " + userName); return false; if (!isActionPermittedForFileHierarchy(fs, childStatus, userName, action, true)) { return false;
if (srcFS.getUri().getScheme().equals("hdfs")) { ContentSummary srcContentSummary = srcFS.getContentSummary(src); if (srcContentSummary.getFileCount() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXNUMFILES) && srcContentSummary.getLength() > conf.getLongVar(HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE)) { LOG.info("Source is " + srcContentSummary.getLength() + " bytes. (MAX: " + conf.getLongVar( HiveConf.ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE) + ")"); LOG.info("Source is " + srcContentSummary.getFileCount() + " files. (MAX: " + conf.getLongVar( LOG.info("Launch distributed copy (distcp) job."); triedDistcp = true; copied = distCp(srcFS, Collections.singletonList(src), dst, deleteSource, null, conf, shims);
try { Utilities.FILE_OP_LOGGER.trace("Copying data from {} to {} ", fromPath, toPath); console.printInfo("Copying data from " + fromPath.toString(), " to " + toPath.toString()); FileSystem srcFs = fromPath.getFileSystem(conf); dstFs = toPath.getFileSystem(conf); FileStatus[] srcs = srcFs.globStatus(fromPath, new EximPathFilter()); if (srcs != null && srcs.length == 1 && srcs[0].isDirectory() "Recursing into a single child directory {}", srcs[0].getPath().getName()); srcs = srcFs.listStatus(srcs[0].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER); if (!FileUtils.mkdir(dstFs, toPath, conf)) { console.printError("Cannot make target directory: " + toPath.toString()); return 2; console.printInfo("Copying file: " + oneSrcPathStr); Utilities.FILE_OP_LOGGER.debug("Copying file {} to {}", oneSrcPathStr, toPath); if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath, false, // delete source true, // overwrite destination
toPath = work.getToPath(); console.printInfo("Copying data from " + fromPath.toString(), " to " + toPath.toString()); FileSystem srcFs = fromPath.getFileSystem(conf); dstFs = toPath.getFileSystem(conf); LOG.debug("ReplCopyTask numFiles:" + (srcFiles == null ? "null" : srcFiles.size())); boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); if (!FileUtils.mkdir(dstFs, toPath, inheritPerms, conf)) { console.printError("Cannot make target directory: " + toPath.toString()); return 2; listBW = new BufferedWriter(new OutputStreamWriter(dstFs.create(listPath))); console.printInfo("Copying file: " + oneSrc.getPath().toString()); LOG.debug("Copying file: " + oneSrc.getPath().toString()); actualSrcFs = oneSrc.getPath().getFileSystem(conf); } else { actualSrcFs = srcFs; LOG.debug("ReplCopyTask :cp:" + oneSrc.getPath() + "=>" + toPath); if (!FileUtils.copy(actualSrcFs, oneSrc.getPath(), dstFs, toPath, false, // delete source true, // overwrite destination
List<FileStatus> result = new ArrayList<FileStatus>(); try { FileStatus fileStatus = fs.getFileStatus(path); FileUtils.listStatusRecursively(fs, fileStatus, filter, result); } catch (IOException e) { StringBuilder sb = new StringBuilder(path.toUri().getPath()); for (int i = 0; i < level; i++) { sb.append(Path.SEPARATOR).append("*"); Path pathPattern = new Path(path, sb.toString()); if (!allLevelsBelow) { return Lists.newArrayList(fs.globStatus(pathPattern, filter)); for (FileStatus status : fs.globStatus(pathPattern)) { if (filter.accept(status.getPath())) { results.add(status); if (status.isDirectory()) { queue.add(status); for (FileStatus child : fs.listStatus(status.getPath())) { if (filter.accept(child.getPath())) { results.add(child);
srcFs = srcf.getFileSystem(conf); } catch (IOException e) { LOG.error("Failed to get src fs", e); throw new HiveException(e.getMessage(), e); String configuredOwner = HiveConf.getVar(conf, ConfVars.HIVE_LOAD_DATA_OWNER); destFs.delete(destf, true); LOG.debug("The path " + destf.toString() + " is deleted"); if (isSrcLocal) { destFs.copyFromLocalFile(srcf, destf); return true; } else { return FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, true, // delete source final ExecutorService pool = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25) > 0 ? Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Move-Thread-%d").build()) : null; final Path destFile = new Path(destf, srcStatus.getPath().getName()); "Unable to move source " + srcStatus.getPath() + " to destination " + destFile;
partitionName = Optional.of(FileUtils.makePartName(partitionColumnNames, partitionValues)); Path path = new Path(writeInfo.getWritePath(), fileNameWithExtension); Optional<Long> size; try { size = Optional.of(hdfsEnvironment.getFileSystem(session.getUser(), path, conf).getFileStatus(path).getLen()); path.toString(), schemaName, tableName, writerImplementation, nodeManager.getCurrentNode().getVersion(), nodeManager.getCurrentNode().getHttpUri().getHost(), session.getIdentity().getPrincipal().map(Principal::getName).orElse(null), nodeManager.getEnvironment(), new Path(path.getParent(), ".tmp-sort." + path.getName()), hiveFileWriter, sortBufferSize,
@Test public void testCopyWithDistcp() throws IOException { Path copySrc = new Path("copySrc"); Path copyDst = new Path("copyDst"); HiveConf conf = new HiveConf(TestFileUtils.class); FileSystem mockFs = mock(FileSystem.class); when(mockFs.getUri()).thenReturn(URI.create("hdfs:///")); ContentSummary mockContentSummary = mock(ContentSummary.class); when(mockContentSummary.getFileCount()).thenReturn(Long.MAX_VALUE); when(mockContentSummary.getLength()).thenReturn(Long.MAX_VALUE); when(mockFs.getContentSummary(any(Path.class))).thenReturn(mockContentSummary); HadoopShims shims = mock(HadoopShims.class); when(shims.runDistCp(Collections.singletonList(copySrc), copyDst, conf)).thenReturn(true); Assert.assertTrue(FileUtils.copy(mockFs, copySrc, mockFs, copyDst, false, false, conf, shims)); verify(shims).runDistCp(Collections.singletonList(copySrc), copyDst, conf); }
srcFs = srcf.getFileSystem(conf); } catch (IOException e) { LOG.error("Failed to get dest fs", e); throw new HiveException(e.getMessage(), e); final boolean inheritPerms = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); HdfsUtils.HadoopFileStatus destStatus = null; destFs.delete(destf, true); LOG.debug("The path " + destf.toString() + " is deleted"); if (isSrcLocal) { destFs.copyFromLocalFile(srcf, destf); if (inheritPerms) { HdfsUtils.setFullFileStatus(conf, destStatus, destFs, destf, true); return FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, true, // delete source final ExecutorService pool = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25) > 0 ? Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Move-Thread-%d").build()) : null;
toPath = work.getToPaths()[0]; console.printInfo("Copying data from " + fromPath.toString(), " to " + toPath.toString()); FileSystem srcFs = fromPath.getFileSystem(conf); if (FileUtils.copy( sourceInfo.getSrcFs(), sourceInfo.getSourcePath(), dstFs, toPath, false, false, conf)) { String writeIdString = conf.get(ReplUtils.REPL_CURRENT_TBL_WRITE_ID); if (writeIdString == null) { console.printError("ReplCopyTask : Write id is not set in the config by open txn task for migration"); console.printInfo("Copying file: " + oneSrc.getPath().toString()); LOG.debug("ReplCopyTask :cp:{}=>{}", oneSrc.getPath(), toPath); srcFiles.add(new ReplChangeManager.FileInfo(oneSrc.getPath().getFileSystem(conf), if (work.getDeleteDestIfExist() && dstFs.exists(toPath)) { LOG.debug(" path " + toPath + " is cleaned before renaming"); hiveDb.cleanUpOneDirectoryForReplace(toPath, dstFs, HIDDEN_FILES_PATH_FILTER, conf, work.getNeedRecycle(), if (!FileUtils.mkdir(dstFs, toPath, conf)) { if (dstFs.exists(destFile)) { if (dstFs.exists(newDestFile)) {
List<FileStatus> result = new ArrayList<FileStatus>(); try { FileStatus fileStatus = fs.getFileStatus(path); FileUtils.listStatusRecursively(fs, fileStatus, result); } catch (IOException e) { StringBuilder sb = new StringBuilder(path.toUri().getPath()); for (int i = 0; i < level; i++) { sb.append(Path.SEPARATOR).append("*"); Path pathPattern = new Path(path, sb.toString()); return fs.globStatus(pathPattern, FileUtils.HIDDEN_FILES_PATH_FILTER);
final String fullname = sourcePath.getName(); final String name = FilenameUtils.getBaseName(sourcePath.getName()); final String type = FilenameUtils.getExtension(sourcePath.getName()); Path destFilePath = new Path(destDirPath, fullname); while (destFs.exists(destFilePath)) { destFilePath = new Path(destDirPath, name + ("_copy_" + counter) + (!type.isEmpty() ? "." + type : "")); counter++; while (!destFs.rename(sourcePath, destFilePath)) { destFilePath = new Path(destDirPath, name + ("_copy_" + counter) + (!type.isEmpty() ? "." + type : "")); counter++; destFs.copyFromLocalFile(sourcePath, destFilePath); } else { FileUtils.copy(sourceFs, sourcePath, destFs, destFilePath, true, // delete source false, // overwrite destination
boolean inheritPerms = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); HadoopShims shims = ShimLoader.getHadoopShims(); if (inheritPerms || replace) { try{ destStatus = shims.getFullFileStatus(conf, fs, destf.getParent()); LOG.debug("The path " + destf.toString() + " is deleted"); fs.delete(destf, true); destStatus = shims.getFullFileStatus(conf, fs, destf.getParent()); success = FileUtils.copy(srcf.getFileSystem(conf), srcf, destf.getFileSystem(conf), destf, true, // delete source success = FileUtils.copy(srcf.getFileSystem(conf), status.getPath(), destf.getFileSystem(conf), destf, true, // delete source throw new HiveException("Unable to move source " + status.getPath() + " to destination " + destf); success = fs.rename(srcf, destf); + ", dest: " + destf.toString() + ", Status:" + success); } catch (IOException ioe) { throw new HiveException("Unable to move source " + srcf + " to destination " + destf, ioe);
if (msckDesc.isRepairPartitions() && !partsNotInMs.isEmpty()) { AbstractList<String> vals = null; String settingStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_MSCK_PATH_VALIDATION); boolean doValidate = !("ignore".equals(settingStr)); boolean doSkip = doValidate && "skip".equals(settingStr); vals = Warehouse.makeValsFromName(part.getPartitionName(), vals); } catch (MetaException ex) { throw new HiveException(ex); String escapedPath = FileUtils.escapePathName(val); assert escapedPath != null; if (escapedPath.equals(val)) continue; iter.remove(); } else { throw new HiveException(errorMsg); table.getDbName(), table.getTableName(), false); try { int batch_size = conf.getIntVar(ConfVars.HIVE_MSCK_REPAIR_BATCH_SIZE); if (batch_size > 0 && partsNotInMs.size() > batch_size) { int counter = 0;
private Path constructPartialPartPath(Path partialPath, String partKey, Map<String, String> partKVs) { StringBuilder sb = new StringBuilder(FileUtils.escapePathName(partKey)); sb.append("="); sb.append(FileUtils.escapePathName(partKVs.get(partKey))); return new Path(partialPath, sb.toString()); }
final FileSystem fs = path.getFileSystem(conf); stat = fs.getFileStatus(path); } catch (FileNotFoundException e) { FileUtils.checkFileAccessWithImpersonation(fs, stat, FsAction.WRITE, user); FileStatus parStatus = fs.getFileStatus(path.getParent()); if (!shims.hasStickyBit(parStatus.getPermission())) { if (parStatus.getOwner().equals(user)) { return; FileStatus childStatus = fs.getFileStatus(path); if (childStatus.getOwner().equals(user)) { return;
toPath = work.getToPath(); console.printInfo("Copying data from " + fromPath.toString(), " to " + toPath.toString()); FileSystem srcFs = fromPath.getFileSystem(conf); dstFs = toPath.getFileSystem(conf); boolean inheritPerms = conf.getBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); if (!FileUtils.mkdir(dstFs, toPath, inheritPerms, conf)) { console.printError("Cannot make target directory: " + toPath.toString()); return 2; console.printInfo("Copying file: " + oneSrc.getPath().toString()); LOG.debug("Copying file: " + oneSrc.getPath().toString()); if (!FileUtils.copy(srcFs, oneSrc.getPath(), dstFs, toPath, false, // delete source true, // overwrite destination