Refine search
@Override public void commitJob(JobContext context) throws IOException { JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); Path tmpLocation = new Path(conf.get(TMP_LOCATION));//this contains base_xxx or delta_xxx_yyy Path finalLocation = new Path(conf.get(FINAL_LOCATION)); FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Moving contents of " + tmpLocation.toString() + " to " + finalLocation.toString()); if(!fs.exists(tmpLocation)) { .writingBase(conf.getBoolean(IS_MAJOR, false)) .isCompressed(conf.getBoolean(IS_COMPRESSED, false)) .minimumWriteId(conf.getLong(MIN_TXN, Long.MAX_VALUE)) LOG.info(context.getJobID() + ": " + tmpLocation + " not found. Assuming 0 splits. Creating " + newDeltaDir); fs.mkdirs(newDeltaDir); AcidUtils.OrcAcidVersion.writeVersionFile(newDeltaDir, fs); return; FileStatus[] contents = fs.listStatus(tmpLocation); Path newPath = new Path(finalLocation, fileStatus.getPath().getName()); fs.rename(fileStatus.getPath(), newPath); AcidUtils.OrcAcidVersion.writeVersionFile(newPath, fs);
private static <T> T loadShims(Map<String, String> classMap, Class<T> xface) { String vers = getMajorVersion(); String className = classMap.get(vers); return createShim(className, xface); }
@Override public void commitJob(JobContext context) throws IOException { JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); Path tmpLocation = new Path(conf.get(TMP_LOCATION));//this contains base_xxx or delta_xxx_yyy Path finalLocation = new Path(conf.get(FINAL_LOCATION)); FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Moving contents of " + tmpLocation.toString() + " to " + finalLocation.toString()); FileStatus[] contents = fs.listStatus(tmpLocation);//expect 1 base or delta dir in this list //we have MIN_TXN, MAX_TXN and IS_MAJOR in JobConf so we could figure out exactly what the dir //name is that we want to rename; leave it for another day for (int i = 0; i < contents.length; i++) { Path newPath = new Path(finalLocation, contents[i].getPath().getName()); fs.rename(contents[i].getPath(), newPath); } fs.delete(tmpLocation, true); }
try { Path location = new Path(table.getSd().getLocation()); FileSystem fs = location.getFileSystem(conf); HadoopShims.HdfsEncryptionShim shim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf); if (!shim.isPathEncrypted(location)) { HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location); FileStatus targetStatus = fs.getFileStatus(location); String targetGroup = targetStatus == null ? null : targetStatus.getGroup(); FileUtils.moveToTrash(fs, location, conf, isAutopurge); fs.mkdirs(location); HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false); } else { FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER); if ((statuses != null) && (statuses.length > 0)) { boolean success = Hive.trashFiles(fs, statuses, conf, isAutopurge); if (!success) { throw new HiveException("Error in deleting the contents of " + location.toString());
private static void fileDiff(String datafile, String testdir) throws Exception { String testFileDir = conf.get("test.data.files"); // inbuilt assumption that the testdir has only one output file. Path di_test = new Path(tmppath, testdir); if (!fs.exists(di_test)) { throw new RuntimeException(tmpdir + File.separator + testdir + " does not exist"); } if (!ShimLoader.getHadoopShims().isDirectory(fs.getFileStatus(di_test))) { throw new RuntimeException(tmpdir + File.separator + testdir + " is not a directory"); } FSDataInputStream fi_test = fs.open((fs.listStatus(di_test))[0].getPath()); FileInputStream fi_gold = new FileInputStream(new File(testFileDir,datafile)); if (!Utilities.contentsEqual(fi_gold, fi_test, false)) { LOG.error(di_test.toString() + " does not match " + datafile); assertEquals(false, true); } }
tableContainer.setSerde(keyContext, valueContext); if (fs.exists(folder)) { if (!fs.isDirectory(folder)) { throw new HiveException("Error, not a directory: " + folder); FileStatus[] fileStatuses = fs.listStatus(folder); if (fileStatuses != null && fileStatuses.length > 0) { AbstractSerDe keySerDe = keyContext.getSerDe(); Path filePath = fileStatus.getPath(); if (ShimLoader.getHadoopShims().isDirectory(fileStatus)) { throw new HiveException("Error, not a file: " + filePath); tableContainer.setKey(folder.toString());
@Override public void abortJob(JobContext context, int status) throws IOException { JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); Path tmpLocation = new Path(conf.get(TMP_LOCATION)); FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Removing " + tmpLocation.toString()); fs.delete(tmpLocation, true); } }
int maxDynamicPartitions = jobInfo.getMaxDynamicPartitions(); Path loadPath = new Path(jobInfo.getLocation()); FileSystem fs = loadPath.getFileSystem(context.getConfiguration()); String dynPathSpec = loadPath.toUri().getPath(); dynPathSpec = dynPathSpec.replaceAll("__HIVE_DEFAULT_PARTITION__", "*"); Path pathPattern = new Path(dynPathSpec); FileStatus[] status = fs.globStatus(pathPattern, FileUtils.HIDDEN_FILES_PATH_FILTER); LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<String, String>(); if (!customDynamicLocationUsed) { Warehouse.makeSpecFromName(fullPartSpec, st.getPath(), null); } else { HCatFileUtil.getPartKeyValuesForCustomLocation(fullPartSpec, jobInfo, st.getPath().toString()); partitionsDiscoveredByPath.put(st.getPath().toString(), fullPartSpec); JobConf jobConf = (JobConf)context.getConfiguration(); JobContext currContext = HCatMapRedUtil.createJobContext( context.getJobID(), InternalUtil.createReporter(HCatMapRedUtil.createTaskAttemptContext(jobConf, ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptID()))); HCatOutputFormat.configureOutputStorageHandler(currContext, jobInfo, fullPartSpec); contextDiscoveredByPath.put(st.getPath().toString(), currContext);
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) { serializedSize = serializedPlan.length(); planMode = "RPC"; conf.set(planPath.toUri().getPath(), serializedPlan); } else { FileSystem fs = planPath.getFileSystem(conf); try { out = fs.create(planPath); SerializationUtilities.serializePlan(kryo, w, out); out.close(); out = null; long fileLen = fs.getFileStatus(planPath).getLen(); serializedSize = fileLen; planMode = "FILE"; if (useCache && !ShimLoader.getHadoopShims().isLocalMode(conf)) { String uriWithLink = planPath.toUri().toString() + "#" + name; DistributedCache.addCacheFile(new URI(uriWithLink), conf); fs.setReplication(planPath, replication);
HiveConf tempConf = new HiveConf(); Path hConfFilePath = new Path(ctx.getLocalTmpPath(), JOBCONF_FILENAME); OutputStream out = null; Properties deltaP = hconf.getChangedProperties(); boolean hadoopLocalMode = ShimLoader.getHadoopShims().isLocalMode(hconf); String hadoopSysDir = "mapred.system.dir"; String hadoopWorkDir = "mapred.local.dir"; continue; tempConf.set(oneProp, hconf.get(oneProp)); out = FileSystem.getLocal(hconf).create(hConfFilePath); tempConf.writeXml(out); } finally { return " -jobconffile " + hConfFilePath.toString();
Path partitionFile = new Path(tmpPath, ".partitions"); ShimLoader.getHadoopShims().setTotalOrderPartitionFile(job, partitionFile); PartitionKeySampler sampler = new PartitionKeySampler(); FileSystem fs = path.getFileSystem(job); for (FileStatus status : fs.globStatus(new Path(path, ".sampling*"))) { sampler.addSampleFile(status.getPath(), job);
runOptions.shouldModifyManagedTableLocation = false; } else { String curWarehouseRoot = HiveConf.getVar(conf, HiveConf.ConfVars.METASTOREWAREHOUSE); if (arePathsEqual(conf, runOptions.oldWarehouseRoot, curWarehouseRoot)) { LOG.info("oldWarehouseRoot is the same as the current warehouse root {}." runOptions.shouldModifyManagedTableLocation = false; } else { Path oldWhRootPath = new Path(runOptions.oldWarehouseRoot); curWhRootPath = new Path(curWarehouseRoot); FileSystem oldWhRootFs = oldWhRootPath.getFileSystem(conf); FileSystem curWhRootFs = curWhRootPath.getFileSystem(conf); oldWhRootPath = oldWhRootFs.makeQualified(oldWhRootPath); curWhRootPath = curWhRootFs.makeQualified(curWhRootPath); if (!FileUtils.equalsFileSystem(oldWhRootFs, curWhRootFs)) { LOG.info("oldWarehouseRoot {} has a different FS than the current warehouse root {}." if (!isHdfs(oldWhRootFs)) { LOG.info("Warehouse is using non-HDFS FileSystem {}. Disabling shouldModifyManagedTableLocation", oldWhRootFs.getUri()); runOptions.shouldModifyManagedTableLocation = false; } else { encryptionShim = ShimLoader.getHadoopShims().createHdfsEncryptionShim(oldWhRootFs, conf); if (!hasEquivalentEncryption(encryptionShim, oldWhRootPath, curWhRootPath)) { LOG.info("oldWarehouseRoot {} and current warehouse root {} have different encryption zones." + HiveConf.setVar(oldWhConf, HiveConf.ConfVars.METASTOREWAREHOUSE, runOptions.oldWarehouseRoot); oldWh = new Warehouse(oldWhConf);
public SecureCmdDoAs(HiveConf conf) throws HiveException, IOException{ // Get delegation token for user from filesystem and write the token along with // metastore tokens into a file String uname = UserGroupInformation.getLoginUser().getShortUserName(); FileSystem fs = FileSystem.get(conf); Credentials cred = new Credentials(); ShimLoader.getHadoopShims().addDelegationTokens(fs, cred, uname); // ask default fs first for (String uri : conf.getStringCollection("mapreduce.job.hdfs-servers")) { try { ShimLoader.getHadoopShims().addDelegationTokens( FileSystem.get(new URI(uri), conf), cred, uname); } catch (URISyntaxException e) { LOG.warn("Invalid URI in mapreduce.job.hdfs-servers:["+uri+"], ignoring.", e); } } tokenFile = File.createTempFile("hive_hadoop_delegation_token", null); tokenPath = new Path(tokenFile.toURI()); //write credential with token to file cred.writeTokenStorageFile(tokenPath, conf); }
private void writeThenReadByRecordReader(int intervalRecordCount, int writeCount, int splitNumber, long maxSplitSize, CompressionCodec codec) throws IOException, InterruptedException { Path testDir = new Path(System.getProperty("test.tmp.dir", ".") + "/mapred/testsmallfirstsplit"); Path testFile = new Path(testDir, "test_rcfile"); fs.delete(testFile, true); Configuration cloneConf = new Configuration(conf); RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length); jonconf.set("mapred.input.dir", testDir.toString()); JobContext context = new Job(jonconf); HiveConf.setLongVar(context.getConfiguration(), HiveConf.ConfVars.MAPREDMAXSPLITSIZE, maxSplitSize); List<InputSplit> splits = inputFormat.getSplits(context); int readCount = 0; for (int i = 0; i < splits.size(); i++) { TaskAttemptContext tac = ShimLoader.getHadoopShims().getHCatShim().createTaskAttemptContext(jonconf, new TaskAttemptID()); RecordReader<LongWritable, BytesRefArrayWritable> rr = inputFormat.createRecordReader(splits.get(i), tac);
try { ugi = UserGroupInformation.getCurrentUser(); shim = ShimLoader.getHadoopShims().getWebHCatShim(appConf, ugi); Path shimCommonJar = new Path( TempletonUtils.findContainingJar(ShimLoader.class, HIVE_SHIMS_FILENAME_PATTERN)); Path shimCommonSecureJar = new Path( TempletonUtils.findContainingJar(HadoopShimsSecure.class, HIVE_SHIMS_FILENAME_PATTERN)); Path shimJar = new Path( TempletonUtils.findContainingJar(shim.getClass(), HIVE_SHIMS_FILENAME_PATTERN)); try { if (ugi != null) { FileSystem.closeAllForUGI(ugi);
private void doDistCpCopyOnce(FileSystem sourceFs, List<Path> srcList, Path destination, boolean usePrivilegedUser) throws IOException { if (hiveConf.getBoolVar(HiveConf.ConfVars.REPL_ADD_RAW_RESERVED_NAMESPACE)) { srcList = srcList.stream().map(path -> { URI uri = path.toUri(); return new Path(uri.getScheme(), uri.getAuthority(), RAW_RESERVED_VIRTUAL_PATH + uri.getPath()); }).collect(Collectors.toList()); URI destinationUri = destination.toUri(); destination = new Path(destinationUri.getScheme(), destinationUri.getAuthority(), RAW_RESERVED_VIRTUAL_PATH + destinationUri.getPath()); } if (!FileUtils.distCp( sourceFs, // source file system srcList, // list of source paths destination, false, usePrivilegedUser ? copyAsUser : null, hiveConf, ShimLoader.getHadoopShims())) { LOG.error("Distcp failed to copy files: " + srcList + " to destination: " + destination); throw new IOException("Distcp operation failed."); } }
@Override public Object run() throws Exception { FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf()); ShimLoader.getHadoopShims().checkFileAccess(fsAsUser, stat, action); addChildren(fsAsUser, stat.getPath(), children); return null; } });
taskId = originalTaskId = Utilities.getTaskId(hconf); initializeSpecPath(); fs = specPath.getFileSystem(hconf); } else { jc = new JobConf(hconf); jc.getPartitionerClass(), null); .getVar(hconf, HIVE_TEMPORARY_TABLE_STORAGE)); if (isTemporary && fsp != null && tmpStorage != StoragePolicyValue.DEFAULT) { assert !conf.isMmTable(); // Not supported for temp tables. final Path outputPath = fsp.buildTaskOutputTempPath(); StoragePolicyShim shim = ShimLoader.getHadoopShims() .getStoragePolicyShim(fs); if (shim != null) { fs.mkdirs(outputPath); shim.setStoragePolicy(outputPath, tmpStorage); logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS);
final FileSystem fs = path.getFileSystem(conf); stat = fs.getFileStatus(path); } catch (FileNotFoundException e) { HadoopShims shims = ShimLoader.getHadoopShims(); if (!shims.supportStickyBit()) { FileStatus parStatus = fs.getFileStatus(path.getParent()); if (!shims.hasStickyBit(parStatus.getPermission())) { if (parStatus.getOwner().equals(user)) { return; FileStatus childStatus = fs.getFileStatus(path); if (childStatus.getOwner().equals(user)) { return;
private void storeBucketPathMapping(TableScanOperator tsOp, FileStatus[] srcs) { Map<String, Integer> bucketFileNameMapping = new HashMap<String, Integer>(); for (int pos = 0; pos < srcs.length; pos++) { if (ShimLoader.getHadoopShims().isDirectory(srcs[pos])) { throw new RuntimeException("Was expecting '" + srcs[pos].getPath() + "' to be bucket file."); } bucketFileNameMapping.put(srcs[pos].getPath().getName(), pos); } tsOp.getConf().setBucketFileNameMapping(bucketFileNameMapping); }