/** * This method set DataMapJob if configured * * @param conf * @throws IOException */ public static void setDataMapJobIfConfigured(Configuration conf) throws IOException { String className = "org.apache.carbondata.spark.rdd.SparkDataMapJob"; DataMapUtil.setDataMapJob(conf, DataMapUtil.createDataMapJob(className)); }
/** * this method gets the datamapJob and call execute of that job, this will be launched for * distributed CG or FG * @return list of Extended blocklets after pruning */ public static List<ExtendedBlocklet> executeDataMapJob(CarbonTable carbonTable, FilterResolverIntf resolver, List<Segment> validSegments, DataMapExprWrapper dataMapExprWrapper, DataMapJob dataMapJob, List<PartitionSpec> partitionsToPrune) throws IOException { String className = "org.apache.carbondata.core.datamap.DistributableDataMapFormat"; SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, validSegments.get(0).getConfiguration()); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); DistributableDataMapFormat dataMapFormat = createDataMapJob(carbonTable, dataMapExprWrapper, validSegments, invalidSegments, partitionsToPrune, className, false); List<ExtendedBlocklet> prunedBlocklets = dataMapJob.execute(dataMapFormat, resolver); // Apply expression on the blocklets. prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets); return prunedBlocklets; }
throws IOException { String dataMapJobClassName = "org.apache.carbondata.spark.rdd.SparkDataMapJob"; DataMapJob dataMapJob = (DataMapJob) createDataMapJob(dataMapJobClassName); if (dataMapJob == null) { return; createDataMapJob(carbonTable, dataMapExprWrapper, validSegments, invalidSegments, null, className, true); dataMapJob.execute(dataMapFormat, null);