ParsedBase parsedBase = ParsedBase.parseBase(p); if(!isDirUsable(child, parsedBase.getVisibilityTxnId(), aborted, validTxnList)) { return; final long writeId = parsedBase.getWriteId(); if(bestBase.oldestBaseWriteId > writeId) {
public static ParsedBase parseBase(Path path) { String filename = path.getName(); if(!filename.startsWith(BASE_PREFIX)) { throw new IllegalArgumentException(filename + " does not start with " + BASE_PREFIX); } int idxOfv = filename.indexOf(VISIBILITY_PREFIX); if(idxOfv < 0) { return new ParsedBase(Long.parseLong(filename.substring(BASE_PREFIX.length())), path); } return new ParsedBase(Long.parseLong(filename.substring(BASE_PREFIX.length(), idxOfv)), Long.parseLong(filename.substring(idxOfv + VISIBILITY_PREFIX.length())), path); } }
if(mergerOptions.getBaseDir().getName().startsWith(AcidUtils.BASE_PREFIX)) { readerPairOptions = modifyForNonAcidSchemaRead(mergerOptions, AcidUtils.ParsedBase.parseBase(mergerOptions.getBaseDir()).getWriteId(), mergerOptions.getBaseDir());
isCompactedBase(ParsedBase.parseBase(bestBase.oldestBase), fs)) {
.setOldStyle(false) .minimumWriteId(0) .maximumWriteId(ParsedBase.parseBase(bucketFile.getParent()).getWriteId()) .bucket(bucket) .writingBase(true);
new byte[0])); assertEquals(123, AcidUtils.ParsedBase.parseBase(new Path("/tmp/base_000123")).getWriteId()); assertEquals(0, AcidUtils.ParsedBase.parseBase(new Path("/tmp/base_000123")).getVisibilityTxnId()); Path dir = new Path("/tmp/tbl"); AcidOutputFormat.Options opts =
if(isBase || isDelta) { if(isBase) { return new TransactionMetaData(AcidUtils.ParsedBase.parseBase(parent).getWriteId(), parent);
/** * We can only use a 'base' if it doesn't have an open txn (from specific reader's point of view) * A 'base' with open txn in its range doesn't have 'enough history' info to produce a correct * snapshot for this reader. * Note that such base is NOT obsolete. Obsolete files are those that are "covered" by other * files within the snapshot. * A base produced by Insert Overwrite is different. Logically it's a delta file but one that * causes anything written previously to be ignored (hence the overwrite). In this case, base_x * is visible if writeid:x is committed for current reader. */ private static boolean isValidBase(ParsedBase parsedBase, ValidWriteIdList writeIdList, FileSystem fs) throws IOException { if(parsedBase.getWriteId() == Long.MIN_VALUE) { //such base is created by 1st compaction in case of non-acid to acid table conversion //By definition there are no open txns with id < 1. return true; } if(isCompactedBase(parsedBase, fs)) { return writeIdList.isValidBase(parsedBase.getWriteId()); } //if here, it's a result of IOW return writeIdList.isWriteIdValid(parsedBase.getWriteId()); } /**
/** * Returns {@code true} if {@code parsedBase} was created by compaction. * As of Hive 4.0 we can tell if a directory is a result of compaction based on the * presence of {@link AcidUtils#VISIBILITY_PATTERN} suffix. Base directories written prior to * that, have to rely on the {@link MetaDataFile} in the directory. So look at the filename first * since that is the cheaper test.*/ private static boolean isCompactedBase(ParsedBase parsedBase, FileSystem fs) throws IOException { return parsedBase.getVisibilityTxnId() > 0 || MetaDataFile.isCompacted(parsedBase.getBaseDirPath(), fs); } private static void getChildState(FileStatus child, HdfsFileStatusWithId childWithId,