public DictionaryInfo buildDictionary(TblColRef col, IReadableTable inpTable) throws IOException { return buildDictionary(col, inpTable, null); }
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, IReadableTable inpTable) throws IOException { CubeDesc cubeDesc = cubeSeg.getCubeDesc(); if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col)) return null; String builderClass = cubeDesc.getDictionaryBuilderClass(col); DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(col, inpTable, builderClass); saveDictionaryInfo(cubeSeg, col, dictInfo); return dictInfo; }
DictionaryInfo nullInfo = dictMgr.buildDictionary(col, MockupReadableTable.newNonExistTable("/a/path")); assertEquals(null, nullInfo); DictionaryInfo info1 = dictMgr.buildDictionary(col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3")); assertEquals(3, info1.getDictionaryObject().getSize()); DictionaryInfo info2 = dictMgr.buildDictionary(col, MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3")); assertTrue(info1 == info2); DictionaryInfo info3 = dictMgr.buildDictionary(col, MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3")); assertTrue(info1 == info3);
public void buildInvertedIndexDictionary(IISegment iiSeg, String factColumnsPath) throws IOException { logger.info("Start building ii dictionary"); DictionaryManager dictMgr = getDictionaryManager(); IIDesc iiDesc = iiSeg.getIIInstance().getDescriptor(); for (TblColRef column : iiDesc.listAllColumns()) { logger.info("Dealing with column {}", column); if (iiDesc.isMetricsCol(column)) { continue; } DictionaryInfo dict = dictMgr.buildDictionary(iiDesc.getModel(), "true", column, factColumnsPath); iiSeg.putDictResPath(column, dict.getResourcePath()); } saveResource(iiSeg.getIIInstance()); }
public DictionaryInfo buildDictionary(TblColRef col, IReadableTable inpTable) throws IOException { return buildDictionary(col, inpTable, null); }
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException { CubeDesc cubeDesc = cubeSeg.getCubeDesc(); if (!cubeDesc.getRowkey().isUseDictionary(col)) return null; DictionaryManager dictMgr = getDictionaryManager(); DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath); cubeSeg.putDictResPath(col, dictInfo.getResourcePath()); saveResource(cubeSeg.getCubeInstance()); return dictInfo; }
@Test @Ignore("hive not ready") public void basic() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_desc"); TblColRef col = cubeDesc.findColumnRef("DEFAULT.TEST_CATEGORY_GROUPINGS", "META_CATEG_NAME"); DictionaryInfo info1 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null); System.out.println(JsonUtil.writeValueAsIndentString(info1)); DictionaryInfo info2 = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null); System.out.println(JsonUtil.writeValueAsIndentString(info2)); assertTrue(info1.getUuid() == info2.getUuid()); assertTrue(info1 == dictMgr.getDictionaryInfo(info1.getResourcePath())); assertTrue(info2 == dictMgr.getDictionaryInfo(info2.getResourcePath())); assertTrue(info1.getDictionaryObject() == info2.getDictionaryObject()); touchDictValues(info1); }
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, IReadableTable inpTable) throws IOException { CubeDesc cubeDesc = cubeSeg.getCubeDesc(); if (!cubeDesc.getAllColumnsNeedDictionaryBuilt().contains(col)) return null; String builderClass = cubeDesc.getDictionaryBuilderClass(col); DictionaryInfo dictInfo = getDictionaryManager().buildDictionary(col, inpTable, builderClass); saveDictionaryInfo(cubeSeg, col, dictInfo); return dictInfo; }
public DictionaryInfo buildDictionary(CubeSegment cubeSeg, TblColRef col, String factColumnsPath) throws IOException { CubeDesc cubeDesc = cubeSeg.getCubeDesc(); if (!cubeDesc.getRowkey().isUseDictionary(col)) return null; DictionaryManager dictMgr = getDictionaryManager(); DictionaryInfo dictInfo = dictMgr.buildDictionary(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, factColumnsPath); if (dictInfo != null) { cubeSeg.putDictResPath(col, dictInfo.getResourcePath()); saveResource(cubeSeg.getCubeInstance()); } return dictInfo; }