if (getValidateSegmentsToAccess(job.getConfiguration())) { if (!accessStreamingSegments) { List<Segment> validSegments = segments.getValidSegments(); streamSegments = segments.getStreamSegments(); streamSegments = getFilteredSegment(job, streamSegments, true, readCommittedScope); if (validSegments.size() == 0) { getFilteredSegment(job, segments.getValidSegments(), true, readCommittedScope); if (filteredSegmentToAccess.size() == 0) { return getSplitsOfStreaming(job, streamSegments, carbonTable); getFilteredNormalSegments(job, segments.getValidSegments(), getSegmentsToAccess(job, readCommittedScope)); streamSegments = segments.getStreamSegments(); if (filteredNormalSegments.size() == 0) { return getSplitsOfStreaming(job, streamSegments, carbonTable); invalidSegments.addAll(segments.getInvalidSegments()); invalidTimestampsList.addAll(updateStatusManager.getInvalidTimestampRange()); if (invalidSegments.size() > 0) { List<Segment> validAndInProgressSegments = new ArrayList<>(segments.getValidSegments()); validAndInProgressSegments.addAll(segments.getListOfInProgressSegments());
throw e; return new ValidAndInvalidSegmentsInfo(listOfValidSegments, listOfValidUpdatedSegments, listOfInvalidSegments, listOfStreamSegments, listOfInProgressSegments);
Set<Segment> segmentSet = new HashSet<>( new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier(), context.getConfiguration()).getValidAndInvalidSegments().getValidSegments()); if (updateTime != null) { CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, updateTime, true,
getFilteredSegment(job, allSegments.getValidSegments(), false, readCommittedScope);
SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = segmentStatusManager.getValidAndInvalidSegments(); List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments(); if (validSegments.isEmpty()) { return carbonProperties.getFormatVersion();
List<Segment> validSegments = new SegmentStatusManager(table.getAbsoluteTableIdentifier()) .getValidAndInvalidSegments().getValidSegments(); String uniqueId = String.valueOf(System.currentTimeMillis()); List<String> tobeUpdatedSegs = new ArrayList<>();
SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, FileFactory.getConfiguration()); List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments(); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); DataMapExprWrapper dataMapExprWrapper = null; if (DataMapStoreManager.getInstance().getAllDataMap(carbonTable).size() > 0) {
/** * Update the table status file with the dropped partitions information * * @param carbonTable * @param uniqueId * @param toBeUpdatedSegments * @param toBeDeleteSegments * @throws IOException */ public static void commitDropPartitions(CarbonTable carbonTable, String uniqueId, List<String> toBeUpdatedSegments, List<String> toBeDeleteSegments, String uuid) throws IOException { if (toBeDeleteSegments.size() > 0 || toBeUpdatedSegments.size() > 0) { Set<Segment> segmentSet = new HashSet<>( new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier()) .getValidAndInvalidSegments().getValidSegments()); CarbonUpdateUtil.updateTableMetadataStatus(segmentSet, carbonTable, uniqueId, true, Segment.toSegmentList(toBeDeleteSegments, null), Segment.toSegmentList(toBeUpdatedSegments, null), uuid); } }
/** * this method gets the datamapJob and call execute of that job, this will be launched for * distributed CG or FG * @return list of Extended blocklets after pruning */ public static List<ExtendedBlocklet> executeDataMapJob(CarbonTable carbonTable, FilterResolverIntf resolver, List<Segment> validSegments, DataMapExprWrapper dataMapExprWrapper, DataMapJob dataMapJob, List<PartitionSpec> partitionsToPrune) throws IOException { String className = "org.apache.carbondata.core.datamap.DistributableDataMapFormat"; SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, validSegments.get(0).getConfiguration()); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); DistributableDataMapFormat dataMapFormat = createDataMapJob(carbonTable, dataMapExprWrapper, validSegments, invalidSegments, partitionsToPrune, className, false); List<ExtendedBlocklet> prunedBlocklets = dataMapJob.execute(dataMapFormat, resolver); // Apply expression on the blocklets. prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets); return prunedBlocklets; }
/** * This method returns the valid segments attached to the table Identifier. * * @param absoluteTableIdentifier * @return */ public static List<Segment> getValidSegmentList(AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException { SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegments = null; try { validAndInvalidSegments = new SegmentStatusManager(absoluteTableIdentifier).getValidAndInvalidSegments(); } catch (IOException e) { LOGGER.error("Error while getting valid segment list for a table identifier"); throw new IOException(); } return validAndInvalidSegments.getValidSegments(); }
/** * this method will delete the datamap folders during drop datamap * @throws MalformedDataMapCommandException */ private void deleteDatamap() throws MalformedDataMapCommandException { SegmentStatusManager ssm = new SegmentStatusManager(tableIdentifier); try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { deleteDatamapData(segment); } } catch (IOException | RuntimeException ex) { throw new MalformedDataMapCommandException( "drop datamap failed, failed to delete datamap directory"); } }
@Override public void deleteDatamapData() { SegmentStatusManager ssm = new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier()); try { List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments(); for (Segment segment : validSegments) { deleteDatamapData(segment); } } catch (IOException e) { LOGGER.error("drop datamap failed, failed to delete datamap directory"); } }