/** * Each write statement in a transaction creates its own delta dir. * @since 1.3.x */ public static String deltaSubdir(long min, long max, int statementId) { return deltaSubdir(min, max) + "_" + String.format(STATEMENT_DIGITS, statementId); }
/** * Each write statement in a transaction creates its own delta dir. * @since 1.3.x */ public static String deltaSubdir(long min, long max, int statementId) { return deltaSubdir(min, max) + "_" + String.format(STATEMENT_DIGITS, statementId); }
/** * delta dir name after compaction */ String makeDeltaDirNameCompacted(long minTxnId, long maxTxnId) { return AcidUtils.deltaSubdir(minTxnId, maxTxnId); } String makeDeleteDeltaDirNameCompacted(long minTxnId, long maxTxnId) {
/** * Return a base or delta directory string * according to the given "baseDirRequired". */ public static String baseOrDeltaSubdir(boolean baseDirRequired, long min, long max, int statementId) { if (!baseDirRequired) { return deltaSubdir(min, max, statementId); } else { return baseDir(min); } }
/** getFinalDirName that takes into account MM, but not DP, LB or buckets. */ public Path getMergeInputDirName() { Path root = getFinalDirName(); if (isMmTable()) { return new Path(root, AcidUtils.deltaSubdir(tableWriteId, tableWriteId, statementId)); } else { return root; } }
String makeDeltaDirName(long minTxnId, long maxTxnId) { if(minTxnId != maxTxnId) { //covers both streaming api and post compaction style. return makeDeltaDirNameCompacted(minTxnId, maxTxnId); } return useHive130DeltaDirName() ? AcidUtils.deltaSubdir(minTxnId, maxTxnId, 0) : AcidUtils.deltaSubdir(minTxnId, maxTxnId); } /**
/** * Convert the list of begin/end transaction id pairs to a list of delta * directories. Note that there may be multiple delta files for the exact same txn range starting * with 1.3.x; * see {@link org.apache.hadoop.hive.ql.io.AcidUtils#deltaSubdir(long, long, int)} * @param root the root directory * @param deltas list of begin/end transaction id pairs * @return the list of delta paths */ public static Path[] deserializeDeltas(Path root, final List<AcidInputFormat.DeltaMetaData> deltas) throws IOException { List<Path> results = new ArrayList<Path>(deltas.size()); for(AcidInputFormat.DeltaMetaData dmd : deltas) { if(dmd.getStmtIds().isEmpty()) { results.add(new Path(root, deltaSubdir(dmd.getMinTxnId(), dmd.getMaxTxnId()))); continue; } for(Integer stmtId : dmd.getStmtIds()) { results.add(new Path(root, deltaSubdir(dmd.getMinTxnId(), dmd.getMaxTxnId(), stmtId))); } } return results.toArray(new Path[results.size()]); }
break; case KEEP_EXISTING: destPath = new Path(destPath, AcidUtils.deltaSubdir(writeId, writeId, stmtId)); break; case OVERWRITE_EXISTING:
String mmDir = AcidUtils.deltaSubdir(writeId, writeId, stmtId);
private void setLoadFileLocation( final ParseContext pCtx, LoadFileDesc lfd) throws SemanticException { // CTAS; make the movetask's destination directory the table's destination. Long txnIdForCtas = null; int stmtId = 0; // CTAS cannot be part of multi-txn stmt FileSinkDesc dataSinkForCtas = null; String loc = null; if (pCtx.getQueryProperties().isCTAS()) { CreateTableDesc ctd = pCtx.getCreateTable(); dataSinkForCtas = ctd.getAndUnsetWriter(); txnIdForCtas = ctd.getInitialMmWriteId(); loc = ctd.getLocation(); } else { loc = pCtx.getCreateViewDesc().getLocation(); } Path location = (loc == null) ? getDefaultCtasLocation(pCtx) : new Path(loc); if (txnIdForCtas != null) { dataSinkForCtas.setDirName(location); location = new Path(location, AcidUtils.deltaSubdir(txnIdForCtas, txnIdForCtas, stmtId)); lfd.setSourcePath(location); if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("Setting MM CTAS to " + location); } } if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { Utilities.FILE_OP_LOGGER.trace("Location for LFD is being set to " + location + "; moving from " + lfd.getSourcePath()); } lfd.setTargetDir(location); }
Path deltaDir = new Path(p, AcidUtils.deltaSubdir(ent.getKey(), ent.getKey())); if (execute) { if (!fs.mkdirs(deltaDir)) { Path deltaDir = new Path(p, AcidUtils.deltaSubdir(writeId, writeId)); if (execute) { if (!fs.mkdirs(deltaDir)) {
deleteDeltaSubdir(options.getMinimumWriteId(), options.getMaximumWriteId()) : deltaSubdir(options.getMinimumWriteId(), options.getMaximumWriteId()); } else { options.getMaximumWriteId(), options.getStatementId()) : deltaSubdir(options.getMinimumWriteId(), options.getMaximumWriteId(), options.getStatementId());
deleteDeltaSubdir(options.getMinimumTransactionId(), options.getMaximumTransactionId()) : deltaSubdir(options.getMinimumTransactionId(), options.getMaximumTransactionId()); } else { options.getMaximumTransactionId(), options.getStatementId()) : deltaSubdir(options.getMinimumTransactionId(), options.getMaximumTransactionId(), options.getStatementId());
if (AcidUtils.isTransactionalTable(table) && !replicationSpec.isInReplicationScope()) { String mmSubdir = replace ? AcidUtils.baseDir(writeId) : AcidUtils.deltaSubdir(writeId, writeId, stmtId); destPath = new Path(tgtPath, mmSubdir);
.inspector(inspector).bucket(BUCKET).writingBase(false).minimumWriteId(1) .maximumWriteId(1).finalDestination(root); Path delta1_1_0 = new Path(root, AcidUtils.deltaSubdir( options.getMinimumWriteId(), options.getMaximumWriteId(), options.getStatementId())); Path bucket0 = AcidUtils.createBucketFile(delta1_1_0, BUCKET);
? AcidUtils.baseDir(writeId) : AcidUtils.deltaSubdir(writeId, writeId, stmtId));
? AcidUtils.baseDir(writeId) : AcidUtils.deltaSubdir(writeId, writeId, stmtId));
Path sideFile = new Path(root + "/" + (use130Format ? AcidUtils.deltaSubdir(10,19,0) : AcidUtils.deltaSubdir(10,19)) + "/bucket_00001_flush_length"); assertEquals(true, fs.exists(sideFile)); assertEquals(32, fs.getFileStatus(sideFile).getLen());
replicationSpec.isInReplicationScope(); destPath = useStagingDirectory ? x.getCtx().getExternalTmpPath(tgtLocation) : new Path(tgtLocation, AcidUtils.deltaSubdir(writeId, writeId, stmtId));
directory.getCurrentDirectories().get(0).getPath()); assertEquals(new Path(root, use130Format ? AcidUtils.deltaSubdir(200,200,0) : AcidUtils.deltaSubdir(200,200)), directory.getCurrentDirectories().get(1).getPath());