/** * Get all datamaps of the table for clearing purpose */ public DataMapExprWrapper getAllDataMapsForClear(CarbonTable carbonTable) throws IOException { List<TableDataMap> allDataMapFG = DataMapStoreManager.getInstance().getAllDataMap(carbonTable); DataMapExprWrapper initialExpr = null; if (allDataMapFG.size() > 0) { initialExpr = new DataMapExprWrapperImpl(allDataMapFG.get(0), null); for (int i = 1; i < allDataMapFG.size(); i++) { initialExpr = new AndDataMapExprWrapper(initialExpr, new DataMapExprWrapperImpl(allDataMapFG.get(i), null), null); } } return initialExpr; }
/** * Method to clear the data maps */ public void clearDataMaps() throws IOException { DataMapStoreManager.getInstance().clearDataMaps(absoluteTableIdentifier); }
@Override public void cleanMeta() throws IOException { if (getMainTable() == null) { throw new UnsupportedOperationException("Table need to be specified in index datamaps"); } DataMapStoreManager.getInstance().dropDataMapSchema(getDataMapSchema().getDataMapName()); }
/** * registered in the store manager. * The datamap is created using datamap name, datamap factory class and table identifier. */ // TODO: make it private public TableDataMap createAndRegisterDataMap(CarbonTable table, DataMapSchema dataMapSchema) throws MalformedDataMapCommandException { DataMapFactory dataMapFactory = getDataMapFactoryClass(table, dataMapSchema); return registerDataMap(table, dataMapSchema, dataMapFactory); }
private static DataMapSchema getDataMapSchema(String dataMapName) throws IOException, NoSuchDataMapException { return DataMapStoreManager.getInstance().getDataMapSchema(dataMapName); }
/** * Returns default blocklet datamap * @param carbonTable * @param resolverIntf * @return */ public static DataMapExprWrapper getDefaultDataMap(CarbonTable carbonTable, FilterResolverIntf resolverIntf) { // Return the default datamap if no other datamap exists. return new DataMapExprWrapperImpl( DataMapStoreManager.getInstance().getDefaultDataMap(carbonTable), resolverIntf); }
/** * register all datamap writer for specified table and segment */ public void registerAllWriter(CarbonTable carbonTable, String segmentId, String taskNo, SegmentProperties segmentProperties) { // clear cache in executor side DataMapStoreManager.getInstance() .clearDataMaps(carbonTable.getCarbonTableIdentifier().getTableUniqueName()); List<TableDataMap> tableIndices; try { tableIndices = DataMapStoreManager.getInstance().getAllDataMap(carbonTable); } catch (IOException e) { LOG.error("Error while retrieving datamaps", e); throw new RuntimeException(e); } if (tableIndices != null) { tblIdentifier = carbonTable.getCarbonTableIdentifier(); for (TableDataMap tableDataMap : tableIndices) { // register it only if it is not lazy datamap, for lazy datamap, user // will rebuild the datamap manually if (!tableDataMap.getDataMapSchema().isLazy()) { DataMapFactory factory = tableDataMap.getDataMapFactory(); register(factory, segmentId, taskNo, segmentProperties); } } } }
@Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { distributable = (DataMapDistributableWrapper) inputSplit; // clear the segmentMap and from cache in executor when there are invalid segments if (invalidSegments.size() > 0) { DataMapStoreManager.getInstance().clearInvalidSegments(table, invalidSegments); } TableDataMap tableDataMap = DataMapStoreManager.getInstance() .getDataMap(table, distributable.getDistributable().getDataMapSchema()); if (isJobToClearDataMaps) { // if job is to clear datamaps just clear datamaps from cache and return DataMapStoreManager.getInstance() .clearDataMaps(table.getCarbonTableIdentifier().getTableUniqueName()); // clear the segment properties cache from executor SegmentPropertiesAndSchemaHolder.getInstance() .invalidate(table.getAbsoluteTableIdentifier()); blockletIterator = Collections.emptyIterator(); return; } dataMaps = tableDataMap.getTableDataMaps(distributable.getDistributable()); List<ExtendedBlocklet> blocklets = tableDataMap .prune(dataMaps, distributable.getDistributable(), dataMapExprWrapper.getFilterResolverIntf(distributable.getUniqueId()), partitions); for (ExtendedBlocklet blocklet : blocklets) { blocklet.setDataMapUniqueId(distributable.getUniqueId()); } blockletIterator = blocklets.iterator(); }
TableDataMap blockletMap = DataMapStoreManager.getInstance().getDefaultDataMap(table); boolean refreshNeeded = DataMapStoreManager.getInstance() .getTableSegmentRefresher(getOrCreateCarbonTable(job.getConfiguration())) .isRefreshNeeded(eachSegment, updateStatusManager.getInvalidTimestampRange(eachSegment.getSegmentNo())); DataMapStoreManager.getInstance() .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()), toBeCleanedSegments);
List<TableDataMap> datamaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable); if (!datamaps.isEmpty()) { for (TableDataMap dataMap : datamaps) { DataMapFactory factoryClass = DataMapStoreManager.getInstance().getDataMapFactoryClass( carbonTable, dataMap.getDataMapSchema()); if (factoryClass.willBecomeStale(operation)) {
for (Segment filteredSegment : filteredSegmentToAccess) { boolean refreshNeeded = DataMapStoreManager.getInstance().getTableSegmentRefresher(carbonTable) .isRefreshNeeded(filteredSegment, updateStatusManager.getInvalidTimestampRange(filteredSegment.getSegmentNo())); if (DataMapStoreManager.getInstance().getTableSegmentRefresher(carbonTable) .isRefreshNeeded(segment.getSegmentNo())) { toBeCleanedSegments.add(segment); DataMapStoreManager.getInstance() .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()), toBeCleanedSegments);
@Override public void initMeta(String ctasSqlStatement) throws MalformedDataMapCommandException, IOException { CarbonTable mainTable = getMainTable(); DataMapSchema dataMapSchema = getDataMapSchema(); if (mainTable == null) { throw new MalformedDataMapCommandException( "Parent table is required to create index datamap"); } ArrayList<RelationIdentifier> relationIdentifiers = new ArrayList<>(); RelationIdentifier relationIdentifier = new RelationIdentifier(mainTable.getDatabaseName(), mainTable.getTableName(), mainTable.getTableInfo().getFactTable().getTableId()); relationIdentifiers.add(relationIdentifier); dataMapSchema.setRelationIdentifier(relationIdentifier); dataMapSchema.setParentTables(relationIdentifiers); DataMapStoreManager.getInstance().registerDataMap(mainTable, dataMapSchema, dataMapFactory); DataMapStoreManager.getInstance().saveDataMapSchema(dataMapSchema); }
/** * Return true if MV datamap present in the specified table * @param carbonTable * @return timeseries data map present */ public static boolean hasMVDataMap(CarbonTable carbonTable) throws IOException { List<DataMapSchema> dataMapSchemaList = DataMapStoreManager.getInstance() .getDataMapSchemasOfTable(carbonTable); for (DataMapSchema dataMapSchema : dataMapSchemaList) { if (dataMapSchema.getProviderName().equalsIgnoreCase(MV.toString())) { return true; } } return false; }
invalidTimestampsList.addAll(updateStatusManager.getInvalidTimestampRange()); if (invalidSegments.size() > 0) { DataMapStoreManager.getInstance() .clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()), invalidSegments);
@Override public void cleanData() { CarbonTable mainTable = getMainTable(); if (mainTable == null) { throw new UnsupportedOperationException("Table need to be specified in index datamaps"); } DataMapStoreManager.getInstance().clearDataMap( mainTable.getAbsoluteTableIdentifier(), getDataMapSchema().getDataMapName()); }
public DataMapChooser(CarbonTable carbonTable) throws IOException { this.carbonTable = carbonTable; // read all datamaps for this table and populate CG and FG datamap list List<TableDataMap> visibleDataMaps = DataMapStoreManager.getInstance().getAllVisibleDataMap(carbonTable); Map<String, DataMapStatusDetail> map = DataMapStatusManager.readDataMapStatusMap(); cgDataMaps = new ArrayList<>(visibleDataMaps.size()); fgDataMaps = new ArrayList<>(visibleDataMaps.size()); for (TableDataMap visibleDataMap : visibleDataMaps) { DataMapStatusDetail status = map.get(visibleDataMap.getDataMapSchema().getDataMapName()); if (status != null && status.isEnabled()) { DataMapLevel level = visibleDataMap.getDataMapFactory().getDataMapLevel(); if (level == DataMapLevel.CG) { cgDataMaps.add(visibleDataMap); } else { fgDataMaps.add(visibleDataMap); } } } }
} else if (key.startsWith(CarbonCommonConstants.CARBON_DATAMAP_VISIBLE)) { String[] keyArray = key.split("\\."); isValid = DataMapStoreManager.getInstance().isDataMapExist( keyArray[keyArray.length - 3], keyArray[keyArray.length - 2],
/** * Clear the datamap/datamaps of a table from memory * * @param identifier Table identifier */ public void clearDataMaps(AbsoluteTableIdentifier identifier) { CarbonTable carbonTable = getCarbonTable(identifier); String tableUniqueName = identifier.getCarbonTableIdentifier().getTableUniqueName(); List<TableDataMap> tableIndices = allDataMaps.get(tableUniqueName); if (tableIndices == null) { String keyUsingTablePath = getKeyUsingTablePath(identifier.getTablePath()); if (keyUsingTablePath != null) { tableUniqueName = keyUsingTablePath; tableIndices = allDataMaps.get(tableUniqueName); } } if (null != carbonTable && tableIndices != null) { try { DataMapUtil.executeDataMapJobForClearingDataMaps(carbonTable); } catch (IOException e) { LOGGER.error("clear dataMap job failed", e); // ignoring the exception } } segmentRefreshMap.remove(identifier.uniqueName()); clearDataMaps(tableUniqueName); allDataMaps.remove(tableUniqueName); tablePathMap.remove(tableUniqueName); }
/** * Clear the invalid segments from all the datamaps of the table * @param carbonTable * @param segments */ public void clearInvalidSegments(CarbonTable carbonTable, List<Segment> segments) throws IOException { getDefaultDataMap(carbonTable).clear(segments); List<TableDataMap> allDataMap = getAllDataMap(carbonTable); for (TableDataMap dataMap: allDataMap) { dataMap.clear(segments); } }
String tableUniqueName = table.getCarbonTableIdentifier().getTableUniqueName(); getTableSegmentRefresher(table); List<TableDataMap> tableIndices = allDataMaps.get(tableUniqueName); if (tableIndices == null) { String keyUsingTablePath = getKeyUsingTablePath(table.getTablePath()); if (keyUsingTablePath != null) { tableUniqueName = keyUsingTablePath; blockletDetailsFetcher = (BlockletDetailsFetcher) dataMapFactory; } else { blockletDetailsFetcher = getBlockletDetailsFetcher(table);