public static CoprocessorRowType fromCuboid(CubeSegment seg, Cuboid cuboid) { List<TblColRef> colList = cuboid.getColumns(); TblColRef[] cols = colList.toArray(new TblColRef[colList.size()]); RowKeyColumnIO colIO = new RowKeyColumnIO(seg.getDimensionEncodingMap()); int[] colSizes = new int[cols.length]; for (int i = 0; i < cols.length; i++) { colSizes[i] = colIO.getColumnLength(cols[i]); } return new CoprocessorRowType(cols, colSizes, seg.getRowKeyPreambleSize()); }
protected void fillColumnValue(TblColRef column, int columnLen, String valueStr, byte[] outputValue, int outputValueOffset) { // special null value case if (valueStr == null) { Arrays.fill(outputValue, outputValueOffset, outputValueOffset + columnLen, defaultValue()); return; } colIO.writeColumn(column, valueStr, 0, this.blankByte, outputValue, outputValueOffset); }
private void collectValue(TblColRef col, byte[] valueBytes, int offset, int length) throws IOException { String strValue = colIO.readColumnString(col, valueBytes, offset, length); values.add(strValue); }
public int getColumnLength(TblColRef col) { return colIO.getColumnLength(col); }
public FilterDecorator(IDimensionEncodingMap dimEncMap, FilterConstantsTreatment filterConstantsTreatment) { this.dimEncMap = dimEncMap; this.columnIO = new RowKeyColumnIO(dimEncMap); this.inevaluableColumns = Sets.newHashSet(); this.filterConstantsTreatment = filterConstantsTreatment; }
private String translate(TblColRef column, String v, int roundingFlag) { byte[] value = Bytes.toBytes(v); byte[] id = new byte[columnIO.getColumnLength(column)]; columnIO.writeColumn(column, value, value.length, roundingFlag, Dictionary.NULL, id, 0); return Dictionary.dictIdToString(id, 0, id.length); } }
public void writeColumn(TblColRef column, byte[] value, int valueLen, int roundingFlag, byte dft, byte[] output, int outputOffset) { Dictionary<String> dict = getDictionary(column); int columnLen = getColumnLength(column); // non-dict value if (dict == null) { byte[] valueBytes = padFixLen(columnLen, value); System.arraycopy(valueBytes, 0, output, outputOffset, columnLen); return; } // dict value try { int id = dict.getIdFromValueBytes(value, 0, valueLen, roundingFlag); BytesUtil.writeUnsigned(id, output, outputOffset, dict.getSizeOfId()); } catch (IllegalArgumentException ex) { for (int i = outputOffset; i < outputOffset + columnLen; i++) output[i] = dft; logger.error("Can't translate value " + Bytes.toString(value, 0, valueLen) + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", dft)); } }
public String readColumnString(TblColRef col, byte[] bytes, int bytesLen) { Dictionary<String> dict = getDictionary(col); if (dict == null) { bytes = Bytes.head(bytes, bytesLen); if (isNull(bytes)) { return null; } bytes = removeFixLenPad(bytes, 0); return Bytes.toString(bytes); } else { int id = BytesUtil.readUnsigned(bytes, 0, bytesLen); try { String value = dict.getValueFromId(id); return value; } catch (IllegalArgumentException e) { logger.error("Can't get dictionary value for column " + col.getName() + " (id = " + id + ")"); return ""; } } }
@Override public byte[] encode(String[] values) { byte[] bytes = new byte[this.getBytesLength()]; int offset = getHeaderLength(); for (int i = 0; i < cuboid.getColumns().size(); i++) { TblColRef column = cuboid.getColumns().get(i); int colLength = colIO.getColumnLength(column); fillColumnValue(column, colLength, values[i], bytes, offset); offset += colLength; } //fill shard and cuboid fillHeader(bytes); return bytes; }
public RowKeyDecoder(CubeSegment cubeSegment) { this.cubeDesc = cubeSegment.getCubeDesc(); this.rowKeySplitter = new RowKeySplitter(cubeSegment); this.colIO = new RowKeyColumnIO(cubeSegment.getDimensionEncodingMap()); this.values = new ArrayList<String>(); }
public void writeColumn(TblColRef column, byte[] value, int valueLen, int roundingFlag, byte dft, byte[] output, int outputOffset) { Dictionary<String> dict = getDictionary(column); int columnLen = getColumnLength(column); // non-dict value if (dict == null) { byte[] valueBytes = padFixLen(columnLen, value); System.arraycopy(valueBytes, 0, output, outputOffset, columnLen); return; } // dict value try { int id = dict.getIdFromValueBytes(value, 0, valueLen, roundingFlag); BytesUtil.writeUnsigned(id, output, outputOffset, dict.getSizeOfId()); } catch (IllegalArgumentException ex) { for (int i = outputOffset; i < outputOffset + columnLen; i++) output[i] = dft; logger.error("Can't translate value " + Bytes.toString(value, 0, valueLen) + " to dictionary ID, roundingFlag " + roundingFlag + ". Using default value " + String.format("\\x%02X", dft)); } }
public String readColumnString(TblColRef col, byte[] bytes, int bytesLen) { Dictionary<String> dict = getDictionary(col); if (dict == null) { bytes = Bytes.head(bytes, bytesLen); if (isNull(bytes)) { return null; } bytes = removeFixLenPad(bytes, 0); return Bytes.toString(bytes); } else { int id = BytesUtil.readUnsigned(bytes, 0, bytesLen); try { String value = dict.getValueFromId(id); return value; } catch (IllegalArgumentException e) { logger.error("Can't get dictionary value for column " + col.getName() + " (id = " + id + ")"); return ""; } } }
public RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) { super(cubeSeg, cuboid); enableSharding = cubeSeg.isEnableSharding(); headerLength = cubeSeg.getRowKeyPreambleSize(); Set<TblColRef> shardByColumns = cubeSeg.getCubeDesc().getShardByColumns(); if (shardByColumns.size() > 1) { throw new IllegalStateException("Does not support multiple UHC now"); } colIO = new RowKeyColumnIO(cubeSeg.getDimensionEncodingMap()); for (TblColRef column : cuboid.getColumns()) { if (shardByColumns.contains(column)) { uhcOffset = bodyLength; uhcLength = colIO.getColumnLength(column); } bodyLength += colIO.getColumnLength(column); } }
splitOffsets[i] = offset; TblColRef col = cuboid.getColumns().get(i); int colLength = colIO.getColumnLength(col); ByteArray split = new ByteArray(bytes, offset, colLength); this.splitBuffers[this.bufferSize++] = split;
public RowKeySplitter(CubeSegment cubeSeg, int splitLen, int bytesLen) { this.enableSharding = cubeSeg.isEnableSharding(); this.cubeDesc = cubeSeg.getCubeDesc(); IDimensionEncodingMap dimEncoding = new CubeDimEncMap(cubeSeg); for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) { dimEncoding.get(rowKeyColDesc.getColRef()); } this.colIO = new RowKeyColumnIO(dimEncoding); this.splitBuffers = new ByteArray[splitLen]; this.splitOffsets = new int[splitLen]; this.bufferSize = 0; }
private String translate(TblColRef column, String v, int roundingFlag) { byte[] id = new byte[dimEncMap.get(column).getLengthOfEncoding()]; columnIO.writeColumn(column, v, roundingFlag, DimensionEncoding.NULL, id, 0); return Dictionary.dictIdToString(id, 0, id.length); } }
private void collectValue(TblColRef col, byte[] valueBytes, int length) throws IOException { String strValue = colIO.readColumnString(col, valueBytes, length); values.add(strValue); }
protected RowKeyEncoder(CubeSegment cubeSeg, Cuboid cuboid) { super(cuboid); colIO = new RowKeyColumnIO(cubeSeg); bytesLength = headerLength = RowConstants.ROWKEY_CUBOIDID_LEN; // header for (TblColRef column : cuboid.getColumns()) { bytesLength += colIO.getColumnLength(column); } }
public int getColumnLength(TblColRef col) { return colIO.getColumnLength(col); }
public FilterDecorator(ISegment seg) { this.columnIO = new RowKeyColumnIO(seg); this.unstrictlyFilteredColumns = Sets.newHashSet(); }