private void buildRowKey() { columnMap = new HashMap<>(); shardByColumns = new HashSet<>(); for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; rowKeyColDesc.init(rowkeyColumns.length - i - 1, cubeDesc); columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isShardBy()) { shardByColumns.add(rowKeyColDesc.getColRef()); } } this.fullMask = 0L; for (int i = 0; i < this.rowkeyColumns.length; i++) { int index = rowkeyColumns[i].getBitIndex(); this.fullMask |= 1L << index; } }
/** * Get dimensions that have dictionary */ public Set<TblColRef> getAllDimsHaveDictionary() { Set<TblColRef> result = Sets.newHashSet(); for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) { TblColRef colRef = rowKeyColDesc.getColRef(); if (rowkey.isUseDictionary(colRef)) { result.add(colRef); } } return result; }
private long getQueryFilterMask(Set<TblColRef> filterColumnD) { long filterMask = 0; logger.info("Filter column set for query: %s", filterColumnD); if (filterColumnD.isEmpty() == false) { RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { if (filterColumnD.contains(allColumns[i].getColRef())) { filterMask |= 1L << allColumns[i].getBitIndex(); } } } logger.info("Filter mask is: {0}", filterMask); return filterMask; }
public TblColRef getColumnByBitIndex(int bitIndex) { RowKeyColDesc[] rowKeyColumns = this.getRowkey().getRowKeyColumns(); return rowKeyColumns[rowKeyColumns.length - 1 - bitIndex].getColRef(); }
private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }
private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); mandatoryColumnMask = 0; for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; String column = rowKeyColDesc.getColumn(); rowKeyColDesc.setColumn(column.toUpperCase()); rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); rowKeyColDesc.setColRef(colNameAbbr.get(column)); if (rowKeyColDesc.getColRef() == null) { throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); } columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isMandatory()) { mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); } } }
/** * Get dimensions that have dictionary */ public Set<TblColRef> getAllDimsHaveDictionary() { Set<TblColRef> result = Sets.newHashSet(); for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) { TblColRef colRef = rowKeyColDesc.getColRef(); if (rowkey.isUseDictionary(colRef)) { result.add(colRef); } } return result; }
private void buildRowKey() { columnMap = new HashMap<>(); shardByColumns = new HashSet<>(); for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; rowKeyColDesc.init(rowkeyColumns.length - i - 1, cubeDesc); columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isShardBy()) { shardByColumns.add(rowKeyColDesc.getColRef()); } } this.fullMask = 0L; for (int i = 0; i < this.rowkeyColumns.length; i++) { int index = rowkeyColumns[i].getBitIndex(); this.fullMask |= 1L << index; } }
private long getQueryFilterMask(Set<TblColRef> filterColumnD) { long filterMask = 0; logger.info("Filter column set for query: %s", filterColumnD); if (filterColumnD.isEmpty() == false) { RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { if (filterColumnD.contains(allColumns[i].getColRef())) { filterMask |= 1L << allColumns[i].getBitIndex(); } } } logger.info("Filter mask is: {0}", filterMask); return filterMask; }
public TblColRef getColumnByBitIndex(int bitIndex) { RowKeyColDesc[] rowKeyColumns = this.getRowkey().getRowKeyColumns(); return rowKeyColumns[rowKeyColumns.length - 1 - bitIndex].getColRef(); }
private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cubeDesc.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
private List<TblColRef> translateIdToColumns(long cuboidID) { List<TblColRef> dimesnions = new ArrayList<TblColRef>(); RowKeyColDesc[] allColumns = cube.getRowkey().getRowKeyColumns(); for (int i = 0; i < allColumns.length; i++) { // NOTE: the order of column in list!!! long bitmask = 1L << allColumns[i].getBitIndex(); if ((cuboidID & bitmask) != 0) { TblColRef colRef = allColumns[i].getColRef(); dimesnions.add(colRef); } } return dimesnions; }
private void buildRowKey(Map<String, TblColRef> colNameAbbr) { columnMap = new HashMap<TblColRef, RowKeyColDesc>(); mandatoryColumnMask = 0; for (int i = 0; i < rowkeyColumns.length; i++) { RowKeyColDesc rowKeyColDesc = rowkeyColumns[i]; String column = rowKeyColDesc.getColumn(); rowKeyColDesc.setColumn(column.toUpperCase()); rowKeyColDesc.setBitIndex(rowkeyColumns.length - i - 1); rowKeyColDesc.setColRef(colNameAbbr.get(column)); if (rowKeyColDesc.getColRef() == null) { throw new IllegalArgumentException("Cannot find rowkey column " + column + " in cube " + cubeDesc); } columnMap.put(rowKeyColDesc.getColRef(), rowKeyColDesc); if (rowKeyColDesc.isMandatory()) { mandatoryColumnMask |= 1L << rowKeyColDesc.getBitIndex(); } } }
public List<TblColRef> getAllColumnsNeedDictionary() { List<TblColRef> result = Lists.newArrayList(); for (RowKeyColDesc rowKeyColDesc : rowkey.getRowKeyColumns()) { TblColRef colRef = rowKeyColDesc.getColRef(); if (rowkey.isUseDictionary(colRef)) { result.add(colRef); } } for (MeasureDesc measure : measures) { MeasureType<?> aggrType = measure.getFunction().getMeasureType(); result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction())); } return result; }
public static String generateRedistributeFlatTableStatement(IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) { final String tableName = flatDesc.getTableName(); StringBuilder sql = new StringBuilder(); sql.append("INSERT OVERWRITE TABLE " + quoteIdentifier(tableName) + " SELECT * FROM " + quoteIdentifier(tableName)); if (flatDesc.getClusterBy() != null) { appendClusterStatement(sql, flatDesc.getClusterBy()); } else if (flatDesc.getDistributedBy() != null) { appendDistributeStatement(sql, Lists.newArrayList(flatDesc.getDistributedBy())); } else { int redistColumnCount = cubeDesc.getConfig().getHiveRedistributeColumnCount(); RowKeyColDesc[] rowKeyColDescs = cubeDesc.getRowkey().getRowKeyColumns(); if (rowKeyColDescs.length < redistColumnCount) redistColumnCount = rowKeyColDescs.length; List<TblColRef> redistColumns = Lists.newArrayListWithCapacity(redistColumnCount); for (int i = 0; i < redistColumnCount; i++) { redistColumns.add(rowKeyColDescs[i].getColRef()); } appendDistributeStatement(sql, redistColumns); } return sql.toString(); }
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }