public String[] buildKey(String[] row) { int keySize = flatDesc.getRowKeyColumnIndexes().length; String[] key = new String[keySize]; for (int i = 0; i < keySize; i++) { key[i] = getCell(flatDesc.getRowKeyColumnIndexes()[i], row); } return key; }
final String cell = row.get(flatDesc.getRowKeyColumnIndexes()[i]); if (cell != null) { row_hashcodes[i] = hc.putString(cell).hash().asBytes();
cuboidIdSplit = Arrays.copyOfRange(cuboidIds, start, end); CuboidStatCalculator calculator = new CuboidStatCalculator(i, intermediateTableDesc.getRowKeyColumnIndexes(), cuboidIdSplit, cuboidsBitSetSplit, isUsePutRowKeyToHllNewAlgorithm, cuboidsHLLSplit); cuboidStatCalculators[i] = calculator;
private void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); keyValueBuilder = new KeyValueBuilder(intermediateTableDesc); reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance); tmpbuf = ByteBuffer.allocate(4096); int[] rokeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes(); Long[] cuboidIds = getCuboidIds(cubeSegment); Integer[][] cuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, rokeyColumnIndexes.length); boolean isNewAlgorithm = isUsePutRowKeyToHllNewAlgorithm(cubeDesc); HLLCounter[] cuboidsHLL = getInitCuboidsHLL(cuboidIds.length, cubeDesc.getConfig().getCubeStatsHLLPrecision()); cuboidStatCalculator = new CuboidStatCalculator(rokeyColumnIndexes, cuboidIds, cuboidsBitSet, isNewAlgorithm, cuboidsHLL); allCols = reducerMapping.getAllDimDictCols(); initDictColDeduper(cubeDesc); initColumnIndex(intermediateTableDesc); initialized = true; } }
public String[] buildKey(String[] row) { int keySize = flatDesc.getRowKeyColumnIndexes().length; String[] key = new String[keySize]; for (int i = 0; i < keySize; i++) { key[i] = getCell(flatDesc.getRowKeyColumnIndexes()[i], row); } return key; }
final String cell = row.get(flatDesc.getRowKeyColumnIndexes()[i]); if (cell != null) { row_hashcodes[i] = hc.putString(cell).hash().asBytes();
cuboidIdSplit = Arrays.copyOfRange(cuboidIds, start, end); CuboidStatCalculator calculator = new CuboidStatCalculator(i, intermediateTableDesc.getRowKeyColumnIndexes(), cuboidIdSplit, cuboidsBitSetSplit, isUsePutRowKeyToHllNewAlgorithm, cuboidsHLLSplit); cuboidStatCalculators[i] = calculator;
private void init() { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeJoinedFlatTableEnrich intermediateTableDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); keyValueBuilder = new KeyValueBuilder(intermediateTableDesc); reducerMapping = new FactDistinctColumnsReducerMapping(cubeInstance); tmpbuf = ByteBuffer.allocate(4096); int[] rokeyColumnIndexes = intermediateTableDesc.getRowKeyColumnIndexes(); Long[] cuboidIds = getCuboidIds(cubeSegment); Integer[][] cuboidsBitSet = CuboidUtil.getCuboidBitSet(cuboidIds, rokeyColumnIndexes.length); boolean isNewAlgorithm = isUsePutRowKeyToHllNewAlgorithm(cubeDesc); HLLCounter[] cuboidsHLL = getInitCuboidsHLL(cuboidIds.length, cubeDesc.getConfig().getCubeStatsHLLPrecision()); cuboidStatCalculator = new CuboidStatCalculator(rokeyColumnIndexes, cuboidIds, cuboidsBitSet, isNewAlgorithm, cuboidsHLL); allCols = reducerMapping.getAllDimDictCols(); initDictColDeduper(cubeDesc); initColumnIndex(intermediateTableDesc); initialized = true; } }