/** * Method to check whether there exists any block which does not contain the blocklet info * * @param splitList * @return */ public static boolean isBlockWithoutBlockletInfoExists(List<CarbonInputSplit> splitList) { for (CarbonInputSplit inputSplit : splitList) { if (null == inputSplit.getDetailInfo().getBlockletInfo()) { return true; } } return false; }
public void calculateLength() { long total = 0; if (splitList.size() > 0 && splitList.get(0).getDetailInfo() != null) { Map<String, Long> blockSizes = new HashMap<>(); for (CarbonInputSplit split : splitList) { blockSizes.put(split.getBlockPath(), split.getDetailInfo().getBlockSize()); } for (Map.Entry<String, Long> entry : blockSizes.entrySet()) { total += entry.getValue(); } } else { for (CarbonInputSplit split : splitList) { total += split.getLength(); } } length = total; }
if (((CarbonInputSplit) inputSplit).getDetailInfo().getBlockFooterOffset() == 0L) { FileReader reader = FileFactory.getFileHolder(FileFactory.getFileType(splitPath), taskAttemptContext.getConfiguration()); ByteBuffer buffer = reader .readByteBuffer(FileFactory.getUpdatedFilePath(splitPath), ((CarbonInputSplit) inputSplit).getDetailInfo().getBlockSize() - 8, 8); ((CarbonInputSplit) inputSplit).getDetailInfo().setBlockFooterOffset(buffer.getLong());
public static List<TableBlockInfo> createBlocks(List<CarbonInputSplit> splitList) { List<TableBlockInfo> tableBlockInfoList = new ArrayList<>(); for (CarbonInputSplit split : splitList) { BlockletInfos blockletInfos = new BlockletInfos(split.getNumberOfBlocklets(), 0, split.getNumberOfBlocklets()); try { TableBlockInfo blockInfo = new TableBlockInfo(split.getPath().toString(), split.blockletId, split.getStart(), split.getSegment().toString(), split.getLocations(), split.getLength(), blockletInfos, split.getVersion(), split.getDeleteDeltaFiles()); blockInfo.setDetailInfo(split.getDetailInfo()); blockInfo.setDataMapWriterPath(split.dataMapWritePath); blockInfo.setBlockOffset(split.getDetailInfo().getBlockFooterOffset()); tableBlockInfoList.add(blockInfo); } catch (IOException e) { throw new RuntimeException("fail to get location of split: " + split, e); } } return tableBlockInfoList; }
public static TableBlockInfo getTableBlockInfo(CarbonInputSplit inputSplit) { BlockletInfos blockletInfos = new BlockletInfos(inputSplit.getNumberOfBlocklets(), 0, inputSplit.getNumberOfBlocklets()); try { TableBlockInfo blockInfo = new TableBlockInfo(inputSplit.getPath().toString(), inputSplit.blockletId, inputSplit.getStart(), inputSplit.getSegment().toString(), inputSplit.getLocations(), inputSplit.getLength(), blockletInfos, inputSplit.getVersion(), inputSplit.getDeleteDeltaFiles()); blockInfo.setDetailInfo(inputSplit.getDetailInfo()); blockInfo.setBlockOffset(inputSplit.getDetailInfo().getBlockFooterOffset()); return blockInfo; } catch (IOException e) { throw new RuntimeException("fail to get location of split: " + inputSplit, e); } }
if (((CarbonInputSplit) inputSplit).getDetailInfo().getBlockFooterOffset() == 0L) { FileReader reader = FileFactory.getFileHolder(FileFactory.getFileType(splitPath), context.getConfiguration()); .readByteBuffer(FileFactory.getUpdatedFilePath(splitPath), inputSplit.getLength() - 8, 8); ((CarbonInputSplit) inputSplit).getDetailInfo().setBlockFooterOffset(buffer.getLong());