public static TableBlockInfo getTableBlockInfo(CarbonInputSplit inputSplit) { BlockletInfos blockletInfos = new BlockletInfos(inputSplit.getNumberOfBlocklets(), 0, inputSplit.getNumberOfBlocklets()); try { TableBlockInfo blockInfo = new TableBlockInfo(inputSplit.getPath().toString(), inputSplit.blockletId, inputSplit.getStart(), inputSplit.getSegment().toString(), inputSplit.getLocations(), inputSplit.getLength(), blockletInfos, inputSplit.getVersion(), inputSplit.getDeleteDeltaFiles()); blockInfo.setDetailInfo(inputSplit.getDetailInfo()); blockInfo.setBlockOffset(inputSplit.getDetailInfo().getBlockFooterOffset()); return blockInfo; } catch (IOException e) { throw new RuntimeException("fail to get location of split: " + inputSplit, e); } }
@Override public void readFields(DataInput in) throws IOException { // read all splits int numSplit = in.readInt(); splitList = new ArrayList<>(numSplit); for (int i = 0; i < numSplit; i++) { CarbonInputSplit split = new CarbonInputSplit(); split.readFields(in); splitList.add(split); } int len = in.readInt(); locations = new String[len]; for (int i = 0; i < len; i++) { locations[i] = in.readUTF(); } fileFormat = FileFormat.getByOrdinal(in.readInt()); }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (!(obj instanceof CarbonInputSplit)) { return false; } CarbonInputSplit other = (CarbonInputSplit) obj; return 0 == this.compareTo(other); }
public void calculateLength() { long total = 0; if (splitList.size() > 0 && splitList.get(0).getDetailInfo() != null) { Map<String, Long> blockSizes = new HashMap<>(); for (CarbonInputSplit split : splitList) { blockSizes.put(split.getBlockPath(), split.getDetailInfo().getBlockSize()); } for (Map.Entry<String, Long> entry : blockSizes.entrySet()) { total += entry.getValue(); } } else { for (CarbonInputSplit split : splitList) { total += split.getLength(); } } length = total; }
if (inputSplit instanceof CarbonInputSplit) { String splitPath = ((CarbonInputSplit) inputSplit).getPath().toString(); if (((CarbonInputSplit) inputSplit).getDetailInfo().getBlockFooterOffset() == 0L) { FileReader reader = FileFactory.getFileHolder(FileFactory.getFileType(splitPath), taskAttemptContext.getConfiguration()); ByteBuffer buffer = reader .readByteBuffer(FileFactory.getUpdatedFilePath(splitPath), ((CarbonInputSplit) inputSplit).getDetailInfo().getBlockSize() - 8, 8); ((CarbonInputSplit) inputSplit).getDetailInfo().setBlockFooterOffset(buffer.getLong()); throw new RuntimeException("unsupported input split type: " + inputSplit); List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList); queryModel.setTableBlockInfos(tableBlockInfoList); queryModel.setVectorReader(true);
updateStatusManager.getInvalidTimestampRange(inputSplit.getSegmentId()); .isInvalidTableBlock(inputSplit.getSegmentId(), inputSplit.getPath().toString(), invalidBlockVOForSegmentId, updateStatusManager)) { continue; .getDeleteDeltaFilePath(inputSplit.getPath().toString(), inputSplit.getSegmentId()); } catch (Exception e) { throw new IOException(e); inputSplit.setDeleteDeltaFiles(deleteDeltaFilePath); result.add(inputSplit);
throw new RuntimeException("unsupported input split type: " + inputSplit); List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList); queryModel.setTableBlockInfos(tableBlockInfoList); queryModel.setVectorReader(true);
new CarbonInputSplit("null", new Path(carbonFile.getAbsolutePath()), 0, carbonFile.getLength(), carbonFile.getLocations(), FileFormat.COLUMNAR_V3); split.setVersion(ColumnarFormatVersion.V3); BlockletDetailInfo info = new BlockletDetailInfo(); split.setDetailInfo(info); info.setBlockSize(carbonFile.getLength()); info.setVersionNumber(split.getVersion().number()); info.setUseMinMaxForPruning(false); splits.add(split);
double seg2 = Double.parseDouble(other.getSegmentId()); if (seg1 - seg2 < 0) { return -1; String filePath1 = this.getPath().getName(); String filePath2 = other.getPath().getName(); if (CarbonTablePath.isCarbonDataFile(filePath1)) { byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1)
/** * Method to check whether there exists any block which does not contain the blocklet info * * @param splitList * @return */ public static boolean isBlockWithoutBlockletInfoExists(List<CarbonInputSplit> splitList) { for (CarbonInputSplit inputSplit : splitList) { if (null == inputSplit.getDetailInfo().getBlockletInfo()) { return true; } } return false; }
private CarbonInputSplit convertToCarbonInputSplit(ExtendedBlocklet blocklet) throws IOException { CarbonInputSplit split = CarbonInputSplit .from(blocklet.getSegmentId(), blocklet.getBlockletId(), new FileSplit(new Path(blocklet.getPath()), 0, blocklet.getLength(), blocklet.getLocations()), ColumnarFormatVersion.valueOf((short) blocklet.getDetailInfo().getVersionNumber()), blocklet.getDataMapWriterPath()); split.setDetailInfo(blocklet.getDetailInfo()); return split; }
List<String> invalidSegments = split.getAllSplits().get(0).getInvalidSegments(); if (invalidSegments.size() > 0) { queryModel.setInvalidSegmentIds(invalidSegments); split.getAllSplits().get(0).getInvalidTimestampRange(); if ((null != invalidTimestampRangeList) && (invalidTimestampRangeList.size() > 0)) { queryModel.setInvalidBlockForSegmentId(invalidTimestampRangeList);
CarbonTablePath.getMetadataPath(table.getTablePath())); for (CarbonInputSplit split : mbSplit.getAllSplits()) { String segmentId = Segment.getSegment(split.getSegmentId(), loadMetadataDetails).toString(); if (uniqueSegments.get(segmentId) == null) { segments.add(Segment.toSegment(segmentId,
protected FileSplit makeSplit(String segmentId, Path file, long start, long length, String[] hosts, FileFormat fileFormat) { return new CarbonInputSplit(segmentId, file, start, length, hosts, fileFormat); }
if (inputSplit instanceof CarbonInputSplit) { splitList = new ArrayList<>(1); String splitPath = ((CarbonInputSplit) inputSplit).getPath().toString(); if (((CarbonInputSplit) inputSplit).getDetailInfo().getBlockFooterOffset() == 0L) { FileReader reader = FileFactory.getFileHolder(FileFactory.getFileType(splitPath), context.getConfiguration()); .readByteBuffer(FileFactory.getUpdatedFilePath(splitPath), inputSplit.getLength() - 8, 8); ((CarbonInputSplit) inputSplit).getDetailInfo().setBlockFooterOffset(buffer.getLong()); List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList); queryModel.setTableBlockInfos(tableBlockInfoList);
List<TableBlockInfo> list = CarbonInputSplit.createBlocks(mbSplit.getAllSplits()); queryModel.setTableBlockInfos(list); long limit = request.limit();
protected FileSplit makeSplit(String segmentId, Path file, long start, long length, String[] hosts, String[] inMemoryHosts, FileFormat fileFormat) { return new CarbonInputSplit(segmentId, file, start, length, hosts, inMemoryHosts, fileFormat); }
public static List<TableBlockInfo> createBlocks(List<CarbonInputSplit> splitList) { List<TableBlockInfo> tableBlockInfoList = new ArrayList<>(); for (CarbonInputSplit split : splitList) { BlockletInfos blockletInfos = new BlockletInfos(split.getNumberOfBlocklets(), 0, split.getNumberOfBlocklets()); try { TableBlockInfo blockInfo = new TableBlockInfo(split.getPath().toString(), split.blockletId, split.getStart(), split.getSegment().toString(), split.getLocations(), split.getLength(), blockletInfos, split.getVersion(), split.getDeleteDeltaFiles()); blockInfo.setDetailInfo(split.getDetailInfo()); blockInfo.setDataMapWriterPath(split.dataMapWritePath); blockInfo.setBlockOffset(split.getDetailInfo().getBlockFooterOffset()); tableBlockInfoList.add(blockInfo); } catch (IOException e) { throw new RuntimeException("fail to get location of split: " + split, e); } } return tableBlockInfoList; }
public static CarbonInputSplit from(String segmentId, String blockletId, FileSplit split, ColumnarFormatVersion version, String dataMapWritePath) throws IOException { return new CarbonInputSplit(segmentId, blockletId, split.getPath(), split.getStart(), split.getLength(), split.getLocations(), version, null, dataMapWritePath); }