/** * This is smart enough to handle streaming ingest where there could be a * {@link OrcAcidUtils#DELTA_SIDE_FILE_SUFFIX} side file. * @param dataFile - ORC acid data file * @return version property from file if there, * {@link #ORC_ACID_VERSION_DEFAULT} otherwise */ @VisibleForTesting public static int getAcidVersionFromDataFile(Path dataFile, FileSystem fs) throws IOException { FileStatus fileStatus = fs.getFileStatus(dataFile); Reader orcReader = OrcFile.createReader(dataFile, OrcFile.readerOptions(fs.getConf()) .filesystem(fs) //make sure to check for side file in case streaming ingest died .maxLength(getLogicalLength(fs, fileStatus))); if (orcReader.hasMetadataValue(ACID_VERSION_KEY)) { char[] versionChar = UTF8.decode(orcReader.getMetadataValue(ACID_VERSION_KEY)).array(); String version = new String(versionChar); return Integer.valueOf(version); } return ORC_ACID_VERSION_DEFAULT; } /**
long logicalLen = AcidUtils.getLogicalLength(fs, fileStatus); if (logicalLen != 0) { Object fileKey = isDefaultFs ? file.getFileId() : null;
AcidUtils.getLogicalLength(fs, bucket0File); Assert.assertTrue("no " + bucket0, fs.exists(bucket0)); Assert.assertFalse("unexpected " + bucket0SideFile, fs.exists(bucket0SideFile)); AcidUtils.getLogicalLength(fs, bucket0File)); expectedException.expect(IOException.class); expectedException.expectMessage(bucket0SideFile.getName() + " found but is not readable"); AcidUtils.getLogicalLength(fs, bucket0File);
OrcFile.readerOptions(context.conf) .filesystem(fs) .maxLength(AcidUtils.getLogicalLength(fs, file))); orcTail = new OrcTail(orcReader.getFileTail(), orcReader.getSerializedFileFooter(), file.getModificationTime());
Assert.assertTrue("Expected " + lengthFile + " to be non empty. lengh=" + lengthFileSize, lengthFileSize > 0); long logicalLength = AcidUtils.getLogicalLength(fs, stat); long actualLength = stat.getLen(); Assert.assertTrue("", logicalLength == actualLength); Assert.assertTrue("Expected " + lengthFile + " to be non empty. lengh=" + lengthFileSize, lengthFileSize > 0); long logicalLength = AcidUtils.getLogicalLength(fs, stat); long actualLength = stat.getLen(); Assert.assertTrue("", logicalLength <= actualLength);
long length = AcidUtils.getLogicalLength(fs, fs.getFileStatus(deltaFile)); assert length >= 0; Reader deltaReader = OrcFile.createReader(deltaFile, OrcFile.readerOptions(conf).maxLength(length));
Assert.assertTrue("Expected " + lengthFile + " to be non empty. lengh=" + lengthFileSize, lengthFileSize > 0); long logicalLength = AcidUtils.getLogicalLength(fs, stat); long actualLength = stat.getLen(); Assert.assertTrue("", logicalLength == actualLength); Assert.assertTrue("Expected " + lengthFile + " to be non empty. lengh=" + lengthFileSize, lengthFileSize > 0); long logicalLength = AcidUtils.getLogicalLength(fs, stat); long actualLength = stat.getLen(); Assert.assertTrue("", logicalLength <= actualLength);