srcFs = FileSystem.newInstance(p.toUri(), conf); if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) {
if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " + "the destination filesystem. Copying file over to destination staging dir.");
if (!FSHDFSUtils.isSameHdfs(conf, realSrcFs, desFs)) { LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " + "the destination store. Copying file over to destination filesystem.");
FileSystem desFs = desPath.getFileSystem(conf); assertTrue(FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); desPath = new Path("/"); desFs = desPath.getFileSystem(conf); assertTrue(FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); desPath = new Path("/"); desFs = desPath.getFileSystem(conf); assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
if (FSHDFSUtils.isSameHdfs(getConf(), sourceFs, targetFs)) { for (Pair<byte[], String> el : famPaths) { Path hfileStagingPath = null;
@Override public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException { if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { // files are copied so no need to move them back return; } Path p = new Path(srcPath); Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName())); LOG.debug("Moving " + stageP + " back to " + p); if(!fs.rename(stageP, p)) throw new IOException("Failed to move HFile: " + stageP + " to " + p); // restore original permission if (origPermissions.containsKey(srcPath)) { fs.setPermission(p, origPermissions.get(srcPath)); } else { LOG.warn("Can't find previous permission for path=" + srcPath); } }
@Override public String prepareBulkLoad(final byte[] family, final String srcPath) throws IOException { Path p = new Path(srcPath); Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName())); if (srcFs == null) { srcFs = FileSystem.get(p.toUri(), conf); } if(!isFile(p)) { throw new IOException("Path does not reference a file: " + p); } // Check to see if the source and target filesystems are the same if (!FSHDFSUtils.isSameHdfs(conf, srcFs, fs)) { LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " + "the destination filesystem. Copying file over to destination staging dir."); FileUtil.copy(srcFs, p, fs, stageP, false, conf); } else { LOG.debug("Moving " + p + " to " + stageP); FileStatus origFileStatus = fs.getFileStatus(p); origPermissions.put(srcPath, origFileStatus.getPermission()); if(!fs.rename(p, stageP)) { throw new IOException("Failed to move HFile: " + p + " to " + stageP); } } fs.setPermission(stageP, PERM_ALL_ACCESS); return stageP.toString(); }
FileSystem desFs = desPath.getFileSystem(conf); assertTrue(FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); desPath = new Path("/"); desFs = desPath.getFileSystem(conf); assertTrue(FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)); desPath = new Path("/"); desFs = desPath.getFileSystem(conf); assertTrue(!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs));
/** * Bulk load: Add a specified store file to the specified family. * If the source file is on the same different file-system is moved from the * source location to the destination location, otherwise is copied over. * * @param familyName Family that will gain the file * @param srcPath {@link Path} to the file to import * @param seqNum Bulk Load sequence number * @return The destination {@link Path} of the bulk loaded file * @throws IOException */ Path bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum) throws IOException { // Copy the file if it's on another filesystem FileSystem srcFs = srcPath.getFileSystem(conf); FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs; // We can't compare FileSystem instances as equals() includes UGI instance // as part of the comparison and won't work when doing SecureBulkLoad // TODO deal with viewFS if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) { LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " + "the destination store. Copying file over to destination filesystem."); Path tmpPath = createTempName(); FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf); LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath); srcPath = tmpPath; } return commitStoreFile(familyName, srcPath, seqNum, true); }
if (FSHDFSUtils.isSameHdfs(getConf(), fs, targetFs)) { for(Pair<byte[], String> el : famPaths) { Path hfileStagingPath = null;