/** * Converts to segment object * @param segmentId * @return */ public static Segment toSegment(String segmentId) { // SegmentId can be combination of segmentNo and segmentFileName. return toSegment(segmentId, null); }
/** * @param segmentId */ public void setSegmentId(String segmentId) { if (segmentId != null) { this.segment = Segment.toSegment(segmentId); } }
public static List<Segment> toSegmentList(String[] segmentIds, ReadCommittedScope readCommittedScope) { List<Segment> list = new ArrayList<>(segmentIds.length); for (String segmentId : segmentIds) { list.add(toSegment(segmentId, readCommittedScope)); } return list; }
public static List<Segment> toSegmentList(List<String> segmentIds, ReadCommittedScope readCommittedScope) { List<Segment> list = new ArrayList<>(segmentIds.size()); for (String segmentId : segmentIds) { list.add(toSegment(segmentId, readCommittedScope)); } return list; }
public TableBlockInfo(String filePath, long blockOffset, String segmentId, String[] locations, long blockLength, ColumnarFormatVersion version, String[] deletedDeltaFilePath) { this.filePath = FileFactory.getUpdatedFilePath(filePath); this.blockletId = "0"; this.blockOffset = blockOffset; this.segment = Segment.toSegment(segmentId); this.locations = locations; this.blockLength = blockLength; this.version = version; this.deletedDeltaFilePath = deletedDeltaFilePath; }
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations, String[] inMemoryHosts, FileFormat fileFormat) { super(path, start, length, locations, inMemoryHosts); this.segment = Segment.toSegment(segmentId); this.fileFormat = fileFormat; taskId = "0"; bucketId = "0"; blockletId = "0"; numberOfBlocklets = 0; invalidSegments = new ArrayList<>(); version = CarbonProperties.getInstance().getFormatVersion(); }
public CarbonInputSplit(String segmentId, Path path, long start, long length, String[] locations, FileFormat fileFormat) { super(path, start, length, locations); this.segment = Segment.toSegment(segmentId); this.fileFormat = fileFormat; taskId = "0"; bucketId = "0"; blockletId = "0"; numberOfBlocklets = 0; invalidSegments = new ArrayList<>(); version = CarbonProperties.getInstance().getFormatVersion(); }
private CarbonInputSplit(String segmentId, String blockletId, Path path, long start, long length, String[] locations, ColumnarFormatVersion version, String[] deleteDeltaFiles, String dataMapWritePath) { super(path, start, length, locations); this.segment = Segment.toSegment(segmentId); String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(path.getName()); if (taskNo.contains("_")) { taskNo = taskNo.split("_")[0]; } this.taskId = taskNo; this.bucketId = CarbonTablePath.DataFileUtil.getBucketNo(path.getName()); this.blockletId = blockletId; this.invalidSegments = new ArrayList<>(); this.version = version; this.deleteDeltaFiles = deleteDeltaFiles; this.dataMapWritePath = dataMapWritePath; }
/** * For getting the comma separated valid segments for merging. * * @param loadMetadataDetails * @return */ public static List<Segment> getValidSegments(List<LoadMetadataDetails> loadMetadataDetails) { List<Segment> segments = new ArrayList<>(); for (LoadMetadataDetails segment : loadMetadataDetails) { //check if this load is an already merged load. if (null != segment.getMergedLoadName()) { segments.add(Segment.toSegment(segment.getMergedLoadName(), null)); } else { segments.add(Segment.toSegment(segment.getLoadName(), null)); } } return segments; }
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); this.segment = Segment.toSegment(in.readUTF()); this.version = ColumnarFormatVersion.valueOf(in.readShort()); this.bucketId = in.readUTF(); this.blockletId = in.readUTF(); int numInvalidSegment = in.readInt(); invalidSegments = new ArrayList<>(numInvalidSegment); for (int i = 0; i < numInvalidSegment; i++) { invalidSegments.add(in.readUTF()); } int numberOfDeleteDeltaFiles = in.readInt(); deleteDeltaFiles = new String[numberOfDeleteDeltaFiles]; for (int i = 0; i < numberOfDeleteDeltaFiles; i++) { deleteDeltaFiles[i] = in.readUTF(); } boolean detailInfoExists = in.readBoolean(); if (detailInfoExists) { detailInfo = new BlockletDetailInfo(); detailInfo.readFields(in); } boolean dataMapWriterPathExists = in.readBoolean(); if (dataMapWriterPathExists) { dataMapWritePath = in.readUTF(); } }
public static boolean validateRangeOfSegmentList(String segmentId) throws InvalidConfigurationException { String[] values = segmentId.split(","); try { if (values.length == 0) { throw new InvalidConfigurationException( "carbon.input.segments.<database_name>.<table_name> value can't be empty."); } for (String value : values) { if (!value.equalsIgnoreCase("*")) { Segment segment = Segment.toSegment(value, null); Float aFloatValue = Float.parseFloat(segment.getSegmentNo()); if (aFloatValue < 0 || aFloatValue > Float.MAX_VALUE) { throw new InvalidConfigurationException( "carbon.input.segments.<database_name>.<table_name> value range should be greater " + "than 0 and less than " + Float.MAX_VALUE); } } } } catch (NumberFormatException nfe) { throw new InvalidConfigurationException( "carbon.input.segments.<database_name>.<table_name> value range is not valid"); } return true; } /**
String segmentId = Segment.getSegment(split.getSegmentId(), loadMetadataDetails).toString(); if (uniqueSegments.get(segmentId) == null) { segments.add(Segment.toSegment(segmentId, new TableStatusReadCommittedScope(table.getAbsoluteTableIdentifier(), loadMetadataDetails, FileFactory.getConfiguration())));
.indexOf(Segment.toSegment(loadDetail.getLoadName(), null)); if (segmentFileIndex > -1) { loadDetail.setSegmentFile(
String segmentId = Segment.toSegment(blocklet.getSegmentId()).getSegmentNo(); String key = CarbonUpdateUtil.getSegmentBlockNameKey(segmentId, blockName);
.contains(Segment.toSegment(loadMetadata.getLoadName(), null))) { loadMetadata.setSegmentFile(loadMetadata.getLoadName() + "_" + updatedTimeStamp + CarbonTablePath.SEGMENT_EXT);
detail.setSegmentStatus(SegmentStatus.MARKED_FOR_DELETE); } else if (segmentFilesTobeUpdated .contains(Segment.toSegment(detail.getLoadName(), null))) { detail.setSegmentFile( detail.getLoadName() + "_" + newMetaEntry.getUpdateStatusFileName()
segmentFilesToBeUpdated.add(Segment.toSegment(segment.getLoadName(), null));