@Override public void configure(JobConf job) { jc = job; work = (ColumnTruncateWork) Utilities.getMapWork(job); Path specPath = work.getOutputDir(); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); try { fs = specPath.getFileSystem(job); autoDelete = fs.deleteOnExit(outPath); } catch (IOException e) { this.exception = true; throw new RuntimeException(e); } }
private Class<? extends RecordReader> getDefaultRecordReader() throws SemanticException { String name; name = conf.getVar(HiveConf.ConfVars.HIVESCRIPTRECORDREADER); try { return (Class<? extends RecordReader>) Class.forName(name, true, Utilities.getSessionSpecifiedClassLoader()); } catch (ClassNotFoundException e) { throw new SemanticException(e); } }
private static String[] getTempFilesFromConf(Configuration conf) { String addedFiles = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE); if (StringUtils.isNotBlank(addedFiles)) { HiveConf.setVar(conf, ConfVars.HIVEADDEDFILES, addedFiles); } String addedJars = Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR); if (StringUtils.isNotBlank(addedJars)) { HiveConf.setVar(conf, ConfVars.HIVEADDEDJARS, addedJars); } String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS); // need to localize the additional jars and files // we need the directory on hdfs to which we shall put all these files String allFiles = auxJars + "," + addedJars + "," + addedFiles; return allFiles.split(","); }
private static MapWork populateMapWork(JobConf jobConf, String inputName) { MapWork work = null; if (inputName != null) { work = (MapWork) Utilities.getMergeWork(jobConf, inputName); // work can still be null if there is no merge work for this input } if (work == null) { work = Utilities.getMapWork(jobConf); } return work; }
private void updatePaths(Path tp, Path ttp) { String taskId = Utilities.getTaskId(jc); tmpPath = tp; taskTmpPath = ttp; finalPath = new Path(tp, taskId); outPath = new Path(ttp, Utilities.toTempPath(taskId)); }
protected void init(JobConf job) { if (mrwork == null || pathToPartitionInfo == null) { if (HiveConf.getVar(job, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE).equals("tez")) { mrwork = (MapWork) Utilities.getMergeWork(job); if (mrwork == null) { mrwork = Utilities.getMapWork(job); } } else { mrwork = Utilities.getMapWork(job); } pathToPartitionInfo = mrwork.getPathToPartitionInfo(); } }
if ((!conf.getBoolVar(HiveConf.ConfVars.HIVE_INSERT_INTO_EXTERNAL_TABLES)) && (dest_tab.getTableType().equals(TableType.EXTERNAL_TABLE))) { throw new SemanticException( dest_path = new Path(dest_tab.getPath(), dpCtx.getSPPath()); table_desc = Utilities.getTableDesc(dest_tab); outputs.add(output); } catch (HiveException e) { throw new SemanticException(e.getMessage(), e); dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri() .getAuthority(), partPath.toUri().getPath()); table_desc = Utilities.getTableDesc(dest_tab); dest_path); boolean isHiveServerQuery = SessionState.get().isHiveServerQuery(); fileSinkDesc.setHiveServerQuery(isHiveServerQuery); String tName = Utilities.getDbTableName(tableDesc.getTableName())[1]; try { Warehouse wh = new Warehouse(conf);
String hiveJar = conf.getJar(); String hadoopExec = conf.getVar(HiveConf.ConfVars.HADOOPBIN); conf.setVar(ConfVars.HIVEADDEDJARS, Utilities.getResourceFiles(conf, SessionState.ResourceType.JAR)); Path planPath = new Path(ctx.getLocalTmpPath(), "plan.xml"); MapredLocalWork plan = getWork(); LOG.info("Generating plan file " + planPath.toString()); out = FileSystem.getLocal(conf).create(planPath); SerializationUtilities.serializePlan(plan, out); out.close(); String cmdLine = hadoopExec + " jar " + jarCmd + " -localtask -plan " + planPath.toString() + " " + isSilent + " " + hiveConfArgs; String files = Utilities.getResourceFiles(conf, SessionState.ResourceType.FILE); workDir = ctx.getLocalTmpPath().toUri().getPath(); Path p = new Path(f); String target = p.toUri().getPath(); String link = workDir + Path.SEPARATOR + p.getName(); if (FileUtil.symLink(target, link) != 0) { if (SessionState.get() != null) { SessionState.get().addLocalMapRedErrors(getId(), errPrintStream.getOutput());
Path path = getPlanPath(conf, name); LOG.debug("PLAN PATH = {}", path); if (path == null) { // Map/reduce plan may not be generated Kryo kryo = SerializationUtilities.borrowKryo(); try { String engine = HiveConf.getVar(conf, ConfVars.HIVE_EXECUTION_ENGINE); if (engine.equals("spark")) { if (StringUtils.isNotEmpty(addedJars)) { ClassLoader loader = Thread.currentThread().getContextClassLoader(); ClassLoader newLoader = addToClassPath(loader, addedJars.split(";")); Thread.currentThread().setContextClassLoader(newLoader); kryo.setClassLoader(newLoader); final long serializedSize; final String planMode; if (HiveConf.getBoolVar(conf, ConfVars.HIVE_RPC_QUERY_PLAN)) { String planStringPath = path.toUri().getPath(); LOG.debug("Loading plan from string: {}", planStringPath); String planString = conf.getRaw(planStringPath); gWork.getName(), humanReadableByteCount(serializedSize)); gWorkMap.get(conf).put(path, gWork); return gWork;
public JobConf pushProjectionsAndFilters(JobConf jobConf, Path path) throws IOException { updateMrWork(jobConf); // TODO: refactor this in HIVE-6366 final JobConf cloneJobConf = new JobConf(jobConf); final PartitionDesc part = pathToPartitionInfo.get(path); if ((part != null) && (part.getTableDesc() != null)) { Utilities.copyTableJobPropertiesToConf(part.getTableDesc(), cloneJobConf); } pushProjectionsAndFilters(cloneJobConf, path.toString(), path.toUri().getPath()); return cloneJobConf; } }
/** * Initialize the mrwork variable in order to get all the partition and start to update the jobconf * * @param job */ private void init(final JobConf job) { final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN); if (mrwork == null && plan != null && plan.length() > 0) { mrwork = Utilities.getMapRedWork(job); pathToPartitionInfo.clear(); for (final Map.Entry<String, PartitionDesc> entry : mrwork.getPathToPartitionInfo().entrySet()) { pathToPartitionInfo.put(new Path(entry.getKey()).toUri().getPath().toString(), entry.getValue()); } } }
if (abort) { if (!autoDelete || isMmTable) { fs.delete(outPath, true); Utilities.writeMmCommitManifest(Lists.newArrayList(outPath), tmpPath.getParent(), fs, taskId, conf.getWriteId(), conf.getStmtId(), null, false); LOG.info("Merged into " + finalPath + "(" + fss.getLen() + " bytes)."); if (isMmTable) { throw new HiveException("Incompatible files should not happen in MM tables."); Path destDir = finalPath.getParent(); Path destPath = destDir; if (!Utilities.isHiveManagedFile(incompatFile)) { final String taskId = Utilities.getTaskId(jc); Path destFilePath = new Path(destDir, new Path(taskId)); for (int counter = 1; fs.exists(destFilePath); counter++) { Utilities.renameOrMoveFiles(fs, incompatFile, destPath); LOG.info("Moved incompatible file " + incompatFile + " to " + destPath); } catch (HiveException e) { throw new HiveException("Failed to close AbstractFileMergeOperator", e);
conf = new JobConf(MapredLocalTask.class); } else { conf = new JobConf(ExecDriver.class); conf.addResource(new Path(jobConfFileName)); boolean isSilent = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESESSIONSILENT); String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYID, "").trim(); if(queryId.isEmpty()) { queryId = "unknown-" + System.currentTimeMillis(); HiveConf.setVar(conf, HiveConf.ConfVars.HIVEQUERYID, queryId); Path p = new Path(planFileName); FileSystem fs = FileSystem.getLocal(conf); InputStream pathData = fs.open(p); ClassLoader loader = conf.getClassLoader(); if (StringUtils.isNotBlank(libjars)) { loader = Utilities.addToClassPath(loader, StringUtils.split(libjars, ",")); conf.setClassLoader(loader); throw new HiveException(e.getMessage(), e);
job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(NullWritable.class); if(work.getNumMapTasks() != null) { HiveConf.setLongVar(job, HiveConf.ConfVars.MAPREDMINSPLITSIZE, work .getMinSplitSize().longValue()); HiveConf.setVar(job, HiveConf.ConfVars.HIVEINPUTFORMAT, work .getInputformat()); Path tempOutPath = Utilities.toTempPath(outputPath); try { FileSystem fs = tempOutPath.getFileSystem(job); if (!fs.exists(tempOutPath)) { fs.mkdirs(tempOutPath); if (noName && this.getQueryPlan() != null) { int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(), maxlen - 6); Utilities.setMapRedWork(job, mrWork, ctx.getMRTmpPath()); String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR); if (!addedJars.isEmpty()) { job.set("tmpjars", addedJars);
public void testAvoidSplitCombination() throws Exception { Configuration conf = new Configuration(); JobConf job = new JobConf(conf); TableDesc tblDesc = Utilities.defaultTd; tblDesc.setInputFileFormatClass(TestSkipCombineInputFormat.class); PartitionDesc partDesc = new PartitionDesc(tblDesc, null); LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>(); pt.put(new Path("/tmp/testfolder1"), partDesc); pt.put(new Path("/tmp/testfolder2"), partDesc); MapredWork mrwork = new MapredWork(); mrwork.getMapWork().setPathToPartitionInfo(pt); Path mapWorkPath = new Path("/tmp/" + System.getProperty("user.name"), "hive"); Utilities.setMapRedWork(conf, mrwork, mapWorkPath); try { Path[] paths = new Path[2]; paths[0] = new Path("/tmp/testfolder1"); paths[1] = new Path("/tmp/testfolder2"); CombineHiveInputFormat combineInputFormat = ReflectionUtils.newInstance(CombineHiveInputFormat.class, conf); combineInputFormat.pathToPartitionInfo = Utilities.getMapWork(conf).getPathToPartitionInfo(); Set results = combineInputFormat.getNonCombinablePathIndices(job, paths, 2); assertEquals("Should have both path indices in the results set", 2, results.size()); } finally { // Cleanup the mapwork path FileSystem.get(conf).delete(mapWorkPath, true); } }
Path tempOutPath = Utilities.toTempPath(outputPath); FileSystem fs = tempOutPath.getFileSystem(job); if (!fs.exists(tempOutPath)) { fs.mkdirs(tempOutPath); int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH); jobName = Utilities.abbreviate(this.getQueryPlan().getQueryStr(), maxlen - 6); Utilities.setMapWork(job, work, ctx.getMRTmpPath(), true); String pwd = HiveConf.getVar(job, HiveConf.ConfVars.METASTOREPWD); if (pwd != null) { HiveConf.setVar(job, HiveConf.ConfVars.METASTOREPWD, "HIVE"); String addedJars = Utilities.getResourceFiles(job, SessionState.ResourceType.JAR); if (!addedJars.isEmpty()) { String mesg = " with exception '" + Utilities.getNameMessage(e) + "'"; if (rj != null) { mesg = "Ended Job = " + rj.getJobID() + mesg; returnVal = 3; String mesg = "Job Commit failed with exception '" + Utilities.getNameMessage(e) + "'"; console.printError(mesg, "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
private void testTaskIds(String [] taskIds, String expectedAttemptId, String expectedTaskId) { Configuration conf = new JobConf(TestOperators.class); for (String one: taskIds) { conf.set("mapred.task.id", one); String attemptId = Utilities.getTaskId(conf); assertEquals(expectedAttemptId, attemptId); assertEquals(Utilities.getTaskIdFromFilename(attemptId), expectedTaskId); assertEquals(Utilities.getTaskIdFromFilename(attemptId + ".gz"), expectedTaskId); assertEquals(Utilities.getTaskIdFromFilename (Utilities.toTempPath(new Path(attemptId + ".gz")).toString()), expectedTaskId); } }
private void updatePaths(Path tmpPath, Path taskTmpPath) { String taskId = Utilities.getTaskId(jc); this.tmpPath = tmpPath; this.taskTmpPath = taskTmpPath; String inputFile = jc.get(MRJobConfig.MAP_INPUT_FILE); int lastSeparator = inputFile.lastIndexOf(Path.SEPARATOR) + 1; finalPath = new Path(tmpPath, inputFile.substring(lastSeparator)); outPath = new Path(taskTmpPath, Utilities.toTempPath(taskId)); }
aliasList.add("a"); LinkedHashMap<Path, ArrayList<String>> pa = new LinkedHashMap<>(); pa.put(new Path("/tmp/testfolder"), aliasList); pt.put(new Path("/tmp/testfolder"), partDesc); mrwork.getMapWork().setAliasToWork(ao); JobConf job = new JobConf(TestPlan.class); job.set("fs.default.name", "file:///"); Utilities.setMapRedWork(job, mrwork, new Path(System.getProperty("java.io.tmpdir") + File.separator + System.getProperty("user.name") + File.separator + "hive")); MapredWork mrwork2 = Utilities.getMapRedWork(job); Utilities.clearWork(job);
private Path setupTempDirWithSingleOutputFile(Configuration hconf) throws IOException { Path tempDirPath = new Path("file://" + temporaryFolder.newFolder().getAbsolutePath()); Path taskOutputPath = new Path(tempDirPath, Utilities.getTaskId(hconf)); FileSystem.getLocal(hconf).create(taskOutputPath).close(); return tempDirPath; }