/** * This method set DataMapJob if configured * * @param conf * @throws IOException */ public static void setDataMapJobIfConfigured(Configuration conf) throws IOException { String className = "org.apache.carbondata.spark.rdd.SparkDataMapJob"; DataMapUtil.setDataMapJob(conf, DataMapUtil.createDataMapJob(className)); }
/** * this method gets the datamapJob and call execute of that job, this will be launched for * distributed CG or FG * @return list of Extended blocklets after pruning */ public static List<ExtendedBlocklet> executeDataMapJob(CarbonTable carbonTable, FilterResolverIntf resolver, List<Segment> validSegments, DataMapExprWrapper dataMapExprWrapper, DataMapJob dataMapJob, List<PartitionSpec> partitionsToPrune) throws IOException { String className = "org.apache.carbondata.core.datamap.DistributableDataMapFormat"; SegmentStatusManager.ValidAndInvalidSegmentsInfo validAndInvalidSegmentsInfo = getValidAndInvalidSegments(carbonTable, validSegments.get(0).getConfiguration()); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); DistributableDataMapFormat dataMapFormat = createDataMapJob(carbonTable, dataMapExprWrapper, validSegments, invalidSegments, partitionsToPrune, className, false); List<ExtendedBlocklet> prunedBlocklets = dataMapJob.execute(dataMapFormat, resolver); // Apply expression on the blocklets. prunedBlocklets = dataMapExprWrapper.pruneBlocklets(prunedBlocklets); return prunedBlocklets; }
.getProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP, CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT)); DataMapJob dataMapJob = DataMapUtil.getDataMapJob(job.getConfiguration()); List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration()); cgPrunedBlocklets = DataMapUtil.executeDataMapJob(carbonTable, resolver, segmentIds, cgDataMapExprWrapper, dataMapJob, partitionsToPrune); } else { List<ExtendedBlocklet> fgPrunedBlocklets = DataMapUtil.executeDataMapJob(carbonTable, resolver, segmentIds, fgDataMapExprWrapper, dataMapJob, partitionsToPrune);
/** * Clear the datamap/datamaps of a table from memory * * @param identifier Table identifier */ public void clearDataMaps(AbsoluteTableIdentifier identifier) { CarbonTable carbonTable = getCarbonTable(identifier); String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName(); List<TableDataMap> tableIndices = allDataMaps.get(tableUniqueName); if (tableIndices == null) { String keyUsingTablePath = getKeyUsingTablePath(identifier.getTablePath()); if (keyUsingTablePath != null) { tableUniqueName = keyUsingTablePath; tableIndices = allDataMaps.get(tableUniqueName); } } if (null != carbonTable && tableIndices != null) { try { DataMapUtil.executeDataMapJobForClearingDataMaps(carbonTable); } catch (IOException e) { LOGGER.error("clear dataMap job failed", e); // ignoring the exception } } segmentRefreshMap.remove(identifier.uniqueName()); clearDataMaps(tableUniqueName); allDataMaps.remove(tableUniqueName); tablePathMap.remove(tableUniqueName); }
CarbonInputFormat.setColumnProjection(conf, columnProjection); if (dataMapJob != null) { DataMapUtil.setDataMapJob(conf, dataMapJob); } else { setDataMapJobIfConfigured(conf);
throws IOException { String dataMapJobClassName = "org.apache.carbondata.spark.rdd.SparkDataMapJob"; DataMapJob dataMapJob = (DataMapJob) createDataMapJob(dataMapJobClassName); if (dataMapJob == null) { return; getValidAndInvalidSegments(carbonTable, FileFactory.getConfiguration()); List<Segment> validSegments = validAndInvalidSegmentsInfo.getValidSegments(); List<Segment> invalidSegments = validAndInvalidSegmentsInfo.getInvalidSegments(); createDataMapJob(carbonTable, dataMapExprWrapper, validSegments, invalidSegments, null, className, true); dataMapJob.execute(dataMapFormat, null);
/** * Clear the datamap/datamaps of a table from memory and disk * * @param identifier Table identifier */ public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) { CarbonTable carbonTable = getCarbonTable(identifier); String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName(); List<TableDataMap> tableIndices = allDataMaps.get(tableUniqueName); if (tableIndices != null) { int i = 0; for (TableDataMap tableDataMap : tableIndices) { if (carbonTable != null && tableDataMap != null && dataMapName .equalsIgnoreCase(tableDataMap.getDataMapSchema().getDataMapName())) { try { DataMapUtil.executeDataMapJobForClearingDataMaps(carbonTable); tableDataMap.clear(); } catch (IOException e) { LOGGER.error("clear dataMap job failed", e); // ignoring the exception } tableDataMap.deleteDatamapData(); tableIndices.remove(i); break; } i++; } allDataMaps.put(tableUniqueName, tableIndices); } }