protected long getMaxCombinations(CubeDesc cubeDesc) { return cubeDesc.getConfig().getCubeAggrGroupMaxCombination(); } }
private int getStatsThreadNum(int cuboidNum) { int unitNum = cubeDesc.getConfig().getCuboidNumberPerStatsCalculator(); if (unitNum <= 0) { logger.warn("config from getCuboidNumberPerStatsCalculator() " + unitNum + " is should larger than 0"); logger.info("Will use single thread for cuboid statistics calculation"); return 1; } int maxCalculatorNum = cubeDesc.getConfig().getCuboidStatsCalculatorMaxNumber(); int calculatorNum = (cuboidNum - 1) / unitNum + 1; if (calculatorNum > maxCalculatorNum) { calculatorNum = maxCalculatorNum; } return calculatorNum; }
public static CuboidScheduler getInstance(CubeDesc cubeDesc) { String clzName = cubeDesc.getConfig().getCuboidScheduler(); try { Class<? extends CuboidScheduler> clz = ClassUtil.forName(clzName, CuboidScheduler.class); return clz.getConstructor(CubeDesc.class).newInstance(cubeDesc); } catch (Exception e) { throw new RuntimeException(e); } }
public static Cuboid findForMandatory(CubeDesc cube, long cuboidID) { return CuboidManager.getInstance(cube.getConfig()).findMandatoryId(cube, cuboidID); }
/** * Get all valid cuboids for agg group, ignoring padding * @param agg agg group * @return cuboidId list */ public Set<Long> calculateCuboidsForAggGroup(AggregationGroup agg) { Set<Long> cuboidHolder = new HashSet<>(); // build tree structure Set<Long> children = getLowestCuboids(agg); while (!children.isEmpty()) { if (cuboidHolder.size() > cubeDesc.getConfig().getCubeAggrGroupMaxCombination()) { throw new TooManyCuboidException("Holder size larger than kylin.cube.aggrgroup.max-combination"); } cuboidHolder.addAll(children); children = getOnTreeParentsByLayer(children, agg); } return Sets.newHashSet(Iterators.filter(cuboidHolder.iterator(), new Predicate<Long>() { @Override public boolean apply(@Nullable Long cuboidId) { return !cubeDesc.isBlackedCuboid(cuboidId); } })); }
private void enableStreamAggregateIfBeneficial(Cuboid cuboid, Set<TblColRef> groupsD, StorageContext context) { CubeDesc cubeDesc = cuboid.getCubeDesc(); boolean enabled = cubeDesc.getConfig().isStreamAggregateEnabled(); Set<TblColRef> shardByInGroups = Sets.newHashSet(); for (TblColRef col : cubeDesc.getShardByColumns()) { if (groupsD.contains(col)) { shardByInGroups.add(col); } } if (!shardByInGroups.isEmpty()) { enabled = false; logger.debug("Aggregate partition results is not beneficial because shard by columns in groupD: {0}", shardByInGroups); } if (!context.isNeedStorageAggregation()) { enabled = false; logger.debug("Aggregate partition results is not beneficial because no storage aggregation"); } if (enabled) { context.enableStreamAggregate(); } }
public static Cuboid findById(CuboidScheduler cuboidScheduler, long cuboidID) { KylinConfig config = cuboidScheduler.getCubeDesc().getConfig(); return CuboidManager.getInstance(config).findById(cuboidScheduler, cuboidID); }
void init(KylinConfig config) { CubeDesc cubeDesc = CubeDescManager.getInstance(config).getCubeDesc(descName); checkNotNull(cubeDesc, "cube descriptor '%s' (for cube '%s') not found", descName, name); if (cubeDesc.isBroken()) { setStatus(RealizationStatusEnum.DESCBROKEN); logger.error("cube descriptor {} (for cube '{}') is broken", cubeDesc.getResourcePath(), name); logger.error("Errors: {}", cubeDesc.getErrorsAsString()); } else if (getStatus() == RealizationStatusEnum.DESCBROKEN) { setStatus(RealizationStatusEnum.DISABLED); logger.info("cube {} changed from DESCBROKEN to DISABLED", name); } setConfig((KylinConfigExt) cubeDesc.getConfig()); }
@SuppressWarnings("unchecked") public static Map<TblColRef, Dictionary<String>> writeDictionary(CubeSegment cubeSegment, Map<TblColRef, Dictionary<String>> dictionaryMap, long startOffset, long endOffset) { Map<TblColRef, Dictionary<String>> realDictMap = Maps.newHashMap(); for (Map.Entry<TblColRef, Dictionary<String>> entry : dictionaryMap.entrySet()) { final TblColRef tblColRef = entry.getKey(); final Dictionary<String> dictionary = entry.getValue(); IReadableTable.TableSignature signature = new IReadableTable.TableSignature(); signature.setLastModifiedTime(System.currentTimeMillis()); signature.setPath(String.format(Locale.ROOT, "streaming_%s_%s", startOffset, endOffset)); signature.setSize(endOffset - startOffset); DictionaryInfo dictInfo = new DictionaryInfo(tblColRef.getColumnDesc(), tblColRef.getDatatype(), signature); logger.info("writing dictionary for TblColRef:" + tblColRef.toString()); DictionaryManager dictionaryManager = DictionaryManager.getInstance(cubeSegment.getCubeDesc().getConfig()); try { DictionaryInfo realDict = dictionaryManager.trySaveNewDict(dictionary, dictInfo); cubeSegment.putDictResPath(tblColRef, realDict.getResourcePath()); realDictMap.put(tblColRef, (Dictionary<String>) realDict.getDictionaryObject()); } catch (IOException e) { throw new RuntimeException("error save dictionary for column:" + tblColRef, e); } } return realDictMap; }
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
if (this.getConfig().isIgnoreCubeSignatureInconsistency()) { logger.info("Skip checking cube signature"); return true;
allCuboidsHLL[i] = new HLLCounter(cubeDesc.getConfig().getCubeStatsHLLPrecision());
public void init(CubeDesc cubeDesc, RowKeyDesc rowKeyDesc) { this.cubeDesc = cubeDesc; this.isMandatoryOnlyValid = cubeDesc.getConfig().getCubeAggrGroupIsMandatoryOnlyValid(); if (this.includes == null || this.includes.length == 0 || this.selectRule == null) { throw new IllegalStateException("AggregationGroup incomplete"); } normalizeColumnNames(); buildPartialCubeFullMask(rowKeyDesc); buildMandatoryColumnMask(rowKeyDesc); buildJointColumnMask(rowKeyDesc); buildJointDimsMask(); buildHierarchyMasks(rowKeyDesc); buildHierarchyDimsMask(); buildNormalDimsMask(); }
public static CubeInstance create(String cubeName, CubeDesc cubeDesc) { CubeInstance cubeInstance = new CubeInstance(); cubeInstance.setConfig((KylinConfigExt) cubeDesc.getConfig()); cubeInstance.setName(cubeName); cubeInstance.setDisplayName(cubeName); cubeInstance.setDescName(cubeDesc.getName()); cubeInstance.setCreateTimeUTC(System.currentTimeMillis()); cubeInstance.setSegments(new Segments<CubeSegment>()); cubeInstance.setStatus(RealizationStatusEnum.DISABLED); cubeInstance.updateRandomUuid(); return cubeInstance; }
@Test public void test() { KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb"); verifyOverride(baseConfig, cubeDesc.getConfig()); }
@Test public void testProject1() { KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); CubeDesc cubeDesc = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb"); verifyProjectOverride(baseConfig, cubeDesc.getConfig()); }
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) { final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks); LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid); byte[] start = encoder.createBuf(); byte[] end = encoder.createBuf(); encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE); encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start); encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE); encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end); byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning System.arraycopy(end, 0, temp, 0, end.length); end = temp; List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys); KylinConfig config = cubeSeg.getCubeDesc().getConfig(); int hbaseCaching = config.getHBaseScanCacheRows(); int hbaseMaxResultSize = config.getHBaseScanMaxResultSize(); // if (isMemoryHungry(selectedColBlocks)) // hbaseCaching /= 10; return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize); }
private void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); keyValueBuilder = new KeyValueBuilder(intermediateTableDesc); reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance); tmpbuf = ByteBuffer.allocate(4096); int[] rokeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes(); Long[] cuboidIds = getCuboidIds(cubeSegment); Integer[][] cuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, rokeyColumnIndexes.length); boolean isNewAlgorithm = isUsePutRowKeyToHllNewAlgorithm(cubeDesc); HLLCounter[] cuboidsHLL = getInitCuboidsHLL(cuboidIds.length, cubeDesc.getConfig().getCubeStatsHLLPrecision()); cuboidStatCalculator = new CuboidStatCalculator(rokeyColumnIndexes, cuboidIds, cuboidsBitSet, isNewAlgorithm, cuboidsHLL); allCols = reducerMapping.getAllDimDictCols(); initDictColDeduper(cubeDesc); initColumnIndex(intermediateTableDesc); initialized = true; } }
@Test public void testPropertiesHotLoad() throws IOException { KylinConfig baseConfig = KylinConfig.getInstanceFromEnv(); KylinConfig oldCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig(); assertEquals(10, oldCubeDescConfig.getMaxConcurrentJobLimit()); //hot load Properties updateProperty("kylin.job.max-concurrent-jobs", "20"); KylinConfig.getInstanceFromEnv().reloadFromSiteProperties(); CubeDescManager.getInstance(baseConfig).reloadCubeDescLocal("ssb"); //test cubeDescConfig KylinConfig newCubeDescConfig = CubeDescManager.getInstance(baseConfig).getCubeDesc("ssb").getConfig(); assertEquals(20, newCubeDescConfig.getMaxConcurrentJobLimit()); //test cubeConfig KylinConfig newCubeConfig = CubeManager.getInstance(baseConfig).getCube("ssb").getConfig(); assertEquals(20, newCubeConfig.getMaxConcurrentJobLimit()); } }
newCubeDesc.setAggregationGroups(cubeDesc.getAggregationGroups()); newCubeDesc.setOverrideKylinProps(cubeDesc.getOverrideKylinProps()); newCubeDesc.setConfig((KylinConfigExt) cubeDesc.getConfig()); newCubeDesc.setPartitionOffsetStart(cubeDesc.getPartitionOffsetStart()); newCubeDesc.setVersion(cubeDesc.getVersion());