public FSPaths(Path specPath) { tmpPath = Utilities.toTempPath(specPath); taskOutputTempPath = Utilities.toTaskTempPath(specPath); outPaths = new Path[numFiles]; finalPaths = new Path[numFiles]; outWriters = new RecordWriter[numFiles]; updaters = new RecordUpdater[numFiles]; if (isDebugEnabled) { LOG.debug("Created slots for " + numFiles); } stat = new Stat(); }
@Override public void configure(JobConf job) { jc = job; work = (ColumnTruncateWork) Utilities.getMapWork(job); Path specPath = work.getOutputDir(); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); try { fs = specPath.getFileSystem(job); autoDelete = fs.deleteOnExit(outPath); } catch (IOException e) { this.exception = true; throw new RuntimeException(e); } }
@Override public void configure(JobConf job) { jc = job; work = (ColumnTruncateWork) Utilities.getMapWork(job); Path specPath = work.getOutputDir(); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); try { fs = specPath.getFileSystem(job); autoDelete = fs.deleteOnExit(outPath); } catch (IOException e) { this.exception = true; throw new RuntimeException(e); } }
public FSPaths(Path specPath, boolean isMmTable) { this.isMmTable = isMmTable; if (!isMmTable) { tmpPathRoot = Utilities.toTempPath(specPath); taskOutputTempPathRoot = Utilities.toTaskTempPath(specPath); subdirForTxn = null; } else { tmpPathRoot = specPath; taskOutputTempPathRoot = null; // Should not be used. subdirForTxn = AcidUtils.baseOrDeltaSubdir(conf.getInsertOverwrite(), conf.getTableWriteId(), conf.getTableWriteId(), conf.getStatementId()); } if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("new FSPaths for " + numFiles + " files, dynParts = " + bDynParts + " (spec path " + specPath + ")"); } outPaths = new Path[numFiles]; finalPaths = new Path[numFiles]; outWriters = new RecordWriter[numFiles]; updaters = new RecordUpdater[numFiles]; if (LOG.isDebugEnabled()) { LOG.debug("Created slots for " + numFiles); } stat = new Stat(); }
boolean isBlobStorage = BlobStorageUtils.isBlobStorageFileSystem(hconf, fs); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); if (success) { if (!isBlobStorage && fs.exists(tmpPath)) {
Path taskTmpPath = Utilities.toTaskTempPath(specPath); if (success) { FileStatus[] statuses = HiveStatsUtils.getFileStatusRecurse(
Path specPath = conf.getOutputPath(); updatePaths(Utilities.toTempPath(specPath), Utilities.toTaskTempPath(specPath)); try { fs = specPath.getFileSystem(hconf);
updatePaths(specPath, null); } else { updatePaths(Utilities.toTempPath(specPath), Utilities.toTaskTempPath(specPath));
public static void mvFileToFinalPath(Path specPath, Configuration hconf, boolean success, Log log, DynamicPartitionCtx dpCtx, FileSinkDesc conf, Reporter reporter) throws IOException, HiveException { FileSystem fs = specPath.getFileSystem(hconf); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); if (success) { if (fs.exists(tmpPath)) { // remove any tmp file or double-committed output files ArrayList<String> emptyBuckets = Utilities.removeTempOrDuplicateFiles(fs, tmpPath, dpCtx); // create empty buckets if necessary if (emptyBuckets.size() > 0) { createEmptyBuckets(hconf, emptyBuckets, conf, reporter); } // move to the file destination log.info("Moving tmp dir: " + tmpPath + " to: " + specPath); Utilities.renameOrMoveFiles(fs, tmpPath, specPath); } } else { fs.delete(tmpPath, true); } fs.delete(taskTmpPath, true); }
public FSPaths(Path specPath) { tmpPath = Utilities.toTempPath(specPath); taskOutputTempPath = Utilities.toTaskTempPath(specPath); outPaths = new Path[numFiles]; finalPaths = new Path[numFiles]; outWriters = new RecordWriter[numFiles]; updaters = new RecordUpdater[numFiles]; if (isDebugEnabled) { LOG.debug("Created slots for " + numFiles); } stat = new Stat(); }
@Override public void configure(JobConf job) { jc = job; work = (ColumnTruncateWork) Utilities.getMapWork(job); Path specPath = work.getOutputDir(); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); try { fs = specPath.getFileSystem(job); autoDelete = fs.deleteOnExit(outPath); } catch (IOException e) { this.exception = true; throw new RuntimeException(e); } }
Path specPath = conf.getOutputPath(); updatePaths(Utilities.toTempPath(specPath), Utilities.toTaskTempPath(specPath)); try { fs = specPath.getFileSystem(hconf);