private DictionaryManager getDictionaryManager() { return DictionaryManager.getInstance(config); }
@Override protected void doSetup(Context context) throws IOException, InterruptedException { super.doSetup(context); final SerializableConfiguration sConf = new SerializableConfiguration(context.getConfiguration()); final String metaUrl = context.getConfiguration().get(BatchConstants.ARG_META_URL); final String cubeName = context.getConfiguration().get(BatchConstants.ARG_CUBE_NAME); final String segmentIds = context.getConfiguration().get(MergeDictionaryJob.OPTION_MERGE_SEGMENT_IDS.getOpt()); final KylinConfig kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(sConf, metaUrl); final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); final CubeDesc cubeDesc = CubeDescManager.getInstance(kylinConfig).getCubeDesc(cubeInstance.getDescName()); mergingSegments = getMergingSegments(cubeInstance, StringUtil.splitByComma(segmentIds)); tblColRefs = cubeDesc.getAllColumnsNeedDictionaryBuilt().toArray(new TblColRef[0]); dictMgr = DictionaryManager.getInstance(kylinConfig); }
/** * For the new segment, we need to create new dimension dictionaries by merging underlying * dictionaries. (https://issues.apache.org/jira/browse/KYLIN-2457, https://issues.apache.org/jira/browse/KYLIN-2800) * @param cube * @param newSeg * @throws IOException */ private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException { DictionaryManager dictMgr = DictionaryManager.getInstance(conf); CubeDesc cubeDesc = cube.getDescriptor(); for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) { logger.info("Merging fact table dictionary on : " + col); List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>(); for (CubeSegment segment : mergingSegments) { logger.info("Including fact table dictionary of segment : " + segment); if (segment.getDictResPath(col) != null) { DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col)); if (dictInfo != null && !dictInfos.contains(dictInfo)) { dictInfos.add(dictInfo); } else { logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col)); } } } mergeDictionaries(dictMgr, newSeg, dictInfos, col); } }
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
private void init() { kylinConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kylinConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName); dictMgr = DictionaryManager.getInstance(kylinConfig); mergingSegments = getMergingSegments(cubeInstance, segmentIds); } }
ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase); ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs); dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs); dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);
private DictionaryManager getDictionaryManager() { return DictionaryManager.getInstance(config); }
private DictionaryManager getDictionaryManager() { return DictionaryManager.getInstance(config); }
DictionaryManager dstDictMgr = DictionaryManager.getInstance(dstConfig); DictionaryManager srcDicMgr = DictionaryManager.getInstance(srcConfig); DictionaryInfo dictSrc = srcDicMgr.getDictionaryInfo(item);
DictionaryManager dstDictMgr = DictionaryManager.getInstance(dstConfig); DictionaryManager srcDicMgr = DictionaryManager.getInstance(srcConfig); DictionaryInfo dictSrc = srcDicMgr.getDictionaryInfo(item);
DictionaryManager dictMgr = DictionaryManager.getInstance(kylinConfig); Dictionary<String> mergedDict = dictMgr.getDictionary(mergedSeg.getDictResPath(col));
private Boolean checkNeedMerging(TblColRef col) throws IOException { Boolean ret = dictsNeedMerging.get(col); if (ret != null) return ret; else { ret = cubeDesc.getRowkey().isUseDictionary(col); if (ret) { String dictTable = (String) DictionaryManager.getInstance(config).decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; ret = cubeDesc.getFactTable().equalsIgnoreCase(dictTable); } dictsNeedMerging.put(col, ret); return ret; } }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf); cubeName = conf.get(BatchConstants.CFG_CUBE_NAME); cube = CubeManager.getInstance(config).getCube(cubeName); cubeDesc = cube.getDescriptor(); intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId); List<TblColRef> columns = baseCuboid.getColumns(); ArrayList<Integer> factDictCols = new ArrayList<Integer>(); RowKeyDesc rowkey = cubeDesc.getRowkey(); DictionaryManager dictMgr = DictionaryManager.getInstance(config); for (int i = 0; i < columns.size(); i++) { TblColRef col = columns.get(i); if (rowkey.isUseDictionary(col) == false) continue; String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0]; if (cubeDesc.getModel().isFactTable(scanTable)) { factDictCols.add(i); } } this.factDictCols = new int[factDictCols.size()]; for (int i = 0; i < factDictCols.size(); i++) this.factDictCols[i] = factDictCols.get(i); schema = HCatInputFormat.getTableSchema(context.getConfiguration()); }
HashSet<TblColRef> colsNeedMeringDict = new HashSet<TblColRef>(); HashSet<TblColRef> colsNeedCopyDict = new HashSet<TblColRef>(); DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
@Before public void setup() throws Exception { createTestMetadata(); dictMgr = DictionaryManager.getInstance(getTestConfig()); }
@Test public void testBuildSaveDictionary() throws IOException { KylinConfig config = KylinConfig.getInstanceFromEnv(); DictionaryManager dictMgr = DictionaryManager.getInstance(config); DataModelManager metaMgr = DataModelManager.getInstance(config); DataModelDesc model = metaMgr.getDataModelDesc("test_kylin_inner_join_model_desc");
DictionaryManager dictMgr = DictionaryManager.getInstance(config); Dictionary<?> sourceDict = dictMgr.getDictionary(sourceCubeSegment.getDictResPath(col)); Dictionary<?> mergedDict = dictMgr.getDictionary(mergedCubeSegment.getDictResPath(col));
DictionaryManager dstDictMgr = DictionaryManager.getInstance(dstConfig); DictionaryManager srcDicMgr = DictionaryManager.getInstance(srcConfig); DictionaryInfo dictSrc = srcDicMgr.getDictionaryInfo(item);
public TableRecordInfo(IISegment iiSegment) { seg = iiSegment; desc = seg.getIIInstance().getDescriptor(); allColumns = desc.listAllColumns(); nColumns = allColumns.size(); dictionaries = new Dictionary<?>[nColumns]; measureSerializers = new FixedLenMeasureCodec<?>[nColumns]; DictionaryManager dictMgr = DictionaryManager.getInstance(desc.getConfig()); int index = 0; for (TblColRef tblColRef : desc.listAllColumns()) { ColumnDesc col = tblColRef.getColumn(); if (desc.isMetricsCol(index)) { measureSerializers[index] = FixedLenMeasureCodec.get(col.getType()); } else { String dictPath = seg.getDictResPath(tblColRef); try { dictionaries[index] = dictMgr.getDictionary(dictPath); } catch (IOException e) { throw new RuntimeException("dictionary " + dictPath + " does not exist ", e); } } index++; } digest = createDigest(); }
dictionaryManager = DictionaryManager.getInstance(getTestConfig()); lfn = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "LSTG_FORMAT_NAME"); lsi = cube.getDescriptor().findColumnRef("DEFAULT.TEST_KYLIN_FACT", "CAL_DT");