/** * Get lowest (not Cube building level) Cuboids for every Agg group * @return lowest level cuboids */ private Set<Long> getLowestCuboids() { return getOnTreeParents(0L, cubeDesc.getAggregationGroups()); }
private long eliminateHierarchyAggregation(long id) { long finalId = id; for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { long temp = id; List<HierarchyMask> hierarchyMaskList = agg.getHierarchyMasks(); if (hierarchyMaskList != null && hierarchyMaskList.size() > 0) { for (HierarchyMask hierMask : hierarchyMaskList) { long[] allMasks = hierMask.allMasks; for (int i = allMasks.length - 1; i > 0; i--) { long bit = allMasks[i] ^ allMasks[i - 1]; if ((inputID & bit) != 0) { temp &= ~allMasks[i - 1]; if (temp < finalId) finalId = temp; } } } } } return finalId; }
private Set<Long> getOnTreeParents(long child) { List<AggregationGroup> aggrs = Lists.newArrayList(); for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { if (agg.isOnTree(child)) { aggrs.add(agg); } } return getOnTreeParents(child, aggrs); }
@Override public boolean apply(@Nullable Long cuboidId) { if (cuboidId == Cuboid.getBaseCuboidId(cubeDesc)) { return true; } for (AggregationGroup agg : cubeDesc.getAggregationGroups()) { if (agg.isOnTree(cuboidId) && checkDimCap(agg, cuboidId)) { return true; } } return false; } }));
public void validateAggregationGroupsCombination() { int index = 1; for (AggregationGroup agg : getAggregationGroups()) { try { long combination = agg.calculateCuboidCombination(); if (combination > config.getCubeAggrGroupMaxCombination()) { String msg = "Aggregation group " + index + " of Cube Desc " + this.name + " has too many combinations: " + combination + ". Use 'mandatory'/'hierarchy'/'joint' to optimize; or update 'kylin.cube.aggrgroup.max-combination' to a bigger value."; throw new TooManyCuboidException(msg); } } catch (TooManyCuboidException e) { throw e; } catch (Exception e) { throw new IllegalStateException("Unknown error while calculating cuboid number for " + // "Aggregation group " + index + " of Cube Desc " + this.name, e); } index++; } }
@Test public void testBadInit3() throws Exception { thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 1 'includes' dimensions not include all the dimensions:"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3) .toArray(new String[3]); cubeDesc.getAggregationGroups().get(0).setIncludes(temp); cubeDesc.init(getTestConfig()); }
@Test public void testBadDesc1() throws IOException { ValidateContext vContext = new ValidateContext(); CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); String[] temp = Arrays.asList(desc.getAggregationGroups().get(0).getIncludes()).subList(0, 3).toArray(new String[3]); desc.getAggregationGroups().get(0).setIncludes(temp); IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); //vContext.print(System.out); assertEquals(1, vContext.getResults().length); assertEquals("Aggregation group 1 'includes' dimensions not include all the dimensions:[seller_id, META_CATEG_NAME, lstg_format_name, lstg_site_id, slr_segment_cd]", (vContext.getResults()[0].getMessage())); }
@Test public void testBadInit9() throws Exception { String[] strs = new String[] { LSTG_FORMAT_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); thrown.expectMessage( "Aggregation group 1 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID } }; cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { META_CATEG_NAME, LSTG_FORMAT_NAME } }; cubeDesc.init(getTestConfig()); }
@Test public void testGoodDesc2() throws IOException { ValidateContext vContext = new ValidateContext(); CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { // new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "CATEG_LVL2_NAME" } }; IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); //vContext.print(System.out); assertEquals(1, vContext.getResults().length); }
@Test public void testBadInit5() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, META_CATEG_NAME }; cubeDesc.init(getTestConfig()); }
@Test public void testBadInit6() throws Exception { CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().mandatoryDims = new String[] { SELLER_ID, LSTG_FORMAT_NAME }; cubeDesc.init(getTestConfig()); }
@Test public void testBadDesc2() throws IOException { ValidateContext vContext = new ValidateContext(); CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class); desc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { // new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "META_CATEG_NAME", "CATEG_LVL2_NAME" } }; IValidatorRule<CubeDesc> rule = getAggregationGroupRule(); rule.validate(desc, vContext); //vContext.print(System.out); assertEquals(2, vContext.getResults().length); assertEquals("Aggregation group 1 joint dimensions has overlap with more than 1 dimensions in same hierarchy: [CATEG_LVL2_NAME, META_CATEG_NAME]", (vContext.getResults()[0].getMessage())); }
@Test public void testBadInit1() throws Exception { thrown.expect(IllegalStateException.class); thrown.expectMessage("AggregationGroup incomplete"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); String[] temp = null; cubeDesc.getAggregationGroups().get(0).setIncludes(temp); cubeDesc.init(getTestConfig()); }
@Test public void testBadInit7() throws Exception { thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 1 require at least 2 dimensions in a joint"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { LSTG_FORMAT_NAME } }; cubeDesc.init(getTestConfig()); }
@Test public void testBadInit2() throws Exception { thrown.expect(IllegalStateException.class); thrown.expectMessage("AggregationGroup incomplete"); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); SelectRule temp = null; cubeDesc.getAggregationGroups().get(0).setSelectRule(temp); cubeDesc.init(getTestConfig()); }
@Test public void testBadInit11() throws Exception { thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 1 require at least 2 dimensions in a hierarchy."); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME } }; cubeDesc.init(getTestConfig()); }
@Test public void testBadInit8() throws Exception { String[] strs = new String[] { CATEG_LVL2_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); thrown.expectMessage( "Aggregation group 1 hierarchy dimensions overlap with joint dimensions: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; cubeDesc.init(getTestConfig()); }
@Test public void testBadInit10() throws Exception { String[] strs = new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID }; thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 1 a dimension exist in more than one joint: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, SLR_SEGMENT_CD }, new String[] { LSTG_FORMAT_NAME, LSTG_SITE_ID, LEAF_CATEG_ID } }; cubeDesc.init(getTestConfig()); }
@Test public void testBadInit12() throws Exception { String[] strs = new String[] { CATEG_LVL2_NAME, META_CATEG_NAME }; thrown.expect(IllegalStateException.class); thrown.expectMessage("Aggregation group 1 a dimension exist in more than one hierarchy: " + sortStrs(strs)); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); cubeDesc.getAggregationGroups().get(0).getSelectRule().hierarchyDims = new String[][] { new String[] { META_CATEG_NAME, CATEG_LVL2_NAME, CATEG_LVL3_NAME }, new String[] { META_CATEG_NAME, CATEG_LVL2_NAME } }; cubeDesc.init(getTestConfig()); }
@Before public void setUp() throws Exception { this.createTestMetadata(); CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC); AggregationGroup g = cubeDesc.getAggregationGroups().get(0); SELLER_ID = getColInAggrGroup(g, "SELLER_ID"); SLR_SEGMENT_CD = getColInAggrGroup(g, "SLR_SEGMENT_CD"); LSTG_FORMAT_NAME = getColInAggrGroup(g, "LSTG_FORMAT_NAME"); LSTG_SITE_ID = getColInAggrGroup(g, "LSTG_SITE_ID"); META_CATEG_NAME = getColInAggrGroup(g, "META_CATEG_NAME"); CATEG_LVL2_NAME = getColInAggrGroup(g, "CATEG_LVL2_NAME"); CATEG_LVL3_NAME = getColInAggrGroup(g, "CATEG_LVL3_NAME"); LEAF_CATEG_ID = getColInAggrGroup(g, "LEAF_CATEG_ID"); }