public static CoprocessorProjector makeForEndpoint(final TableRecordInfo tableInfo, final Collection<TblColRef> groupby) { byte[] mask = new byte[tableInfo.getDigest().getByteFormLen()]; int maskIdx = 0; for (int i = 0; i < tableInfo.getDigest().getColumnCount(); ++i) { TblColRef tblColRef = tableInfo.getColumns().get(i); int length = tableInfo.getDigest().length(i); byte bits = groupby.contains(tblColRef) ? (byte) 0xff : 0x00; for (int j = 0; j < length; ++j) { mask[maskIdx++] = bits; } } return new CoprocessorProjector(mask); }
public TableRecord createTableRecord() { return new TableRecord(digest.createTableRecordBytes(), this); }
@Override public Slice next() { int columns = 0; ColumnValueContainer[] valueContainers = new ColumnValueContainer[incompleteDigest.getColumnCount()]; Dictionary<?>[] localDictionaries = new Dictionary<?>[incompleteDigest.getColumnCount()]; boolean firstTime = true; short curShard = 0; long lastTimestamp = 0; while (feedingIterator.hasNext() && columns < incompleteDigest.getColumnCount()) { final IIRow row = feedingIterator.next(); final ImmutableBytesWritable key = row.getKey(); if (incompleteDigest.isMetrics(curCol)) { CompressedValueContainer c = new CompressedValueContainer(incompleteDigest, curCol, 0); c.fromBytes(row.getValue()); localDictionaries[curCol] = dictionary; } else { CompressedValueContainer c = new CompressedValueContainer(incompleteDigest.length(curCol), incompleteDigest.getMaxID(curCol) - 0 + 1, 0); c.fromBytes(row.getValue()); valueContainers[curCol] = c; Preconditions.checkArgument(columns == incompleteDigest.getColumnCount(), "column count is " + columns + " should be equals to incompleteDigest.getColumnCount() " + incompleteDigest.getColumnCount()); TableRecordInfoDigest digest = createDigest(columns, incompleteDigest.getIsMetric(), incompleteDigest.getMetricDataTypes(), localDictionaries); Slice slice = new Slice(digest, curShard, curTimestamp, valueContainers); slice.setLocalDictionaries(localDictionaries);
private void addContainer(int col, ColumnValueContainer c) { if (containers == null) { containers = new ColumnValueContainer[info.getColumnCount()]; } containers[col] = c; }
@Test public void testSerialize() { byte[] x = TableRecordInfoDigest.serialize(this.tableRecordInfo.getDigest()); TableRecordInfoDigest d = TableRecordInfoDigest.deserialize(x); assertEquals(d.getColumnCount(), 25); }
public int length(int col) { return digest.length(col); }
public RawTableRecord(TableRecordInfoDigest info) { this.digest = info; this.buf = new byte[info.getByteFormLen()]; reset(); }
@Override public void serialize(EndpointAggregators value, ByteBuffer out) { BytesUtil.writeAsciiStringArray(value.funcNames, out); BytesUtil.writeAsciiStringArray(value.dataTypes, out); BytesUtil.writeVInt(value.metricInfos.length, out); for (int i = 0; i < value.metricInfos.length; ++i) { MetricInfo metricInfo = value.metricInfos[i]; BytesUtil.writeAsciiString(metricInfo.type.toString(), out); BytesUtil.writeVInt(metricInfo.refIndex, out); BytesUtil.writeVInt(metricInfo.precision, out); } BytesUtil.writeByteArray(TableRecordInfoDigest.serialize(value.tableRecordInfoDigest), out); }
private TableRecordInfoDigest createDigest() { // isMetric boolean[] isMetric = new boolean[nColumns]; for (int i = 0; i < nColumns; ++i) { isMetric[i] = desc.isMetricsCol(i); } // lengths int[] lengths = new int[nColumns]; for (int i = 0; i < nColumns; ++i) { lengths[i] = isMetric[i] ? measureSerializers[i].getLength() : dictionaries[i].getSizeOfId(); } // dict max id int[] dictMaxIds = new int[nColumns]; for (int i = 0; i < nColumns; ++i) { if (!isMetric[i]) dictMaxIds[i] = dictionaries[i].getMaxId(); } // offsets int pos = 0; int[] offsets = new int[nColumns]; for (int i = 0; i < nColumns; i++) { offsets[i] = pos; pos += lengths[i]; } int byteFormLen = pos; return new TableRecordInfoDigest(nColumns, byteFormLen, offsets, dictMaxIds, lengths, isMetric, measureSerializers); }
@Override public EndpointAggregators deserialize(ByteBuffer in) { String[] funcNames = BytesUtil.readAsciiStringArray(in); String[] dataTypes = BytesUtil.readAsciiStringArray(in); int metricInfoLength = BytesUtil.readVInt(in); MetricInfo[] infos = new MetricInfo[metricInfoLength]; for (int i = 0; i < infos.length; ++i) { MetricType type = MetricType.valueOf(BytesUtil.readAsciiString(in)); int refIndex = BytesUtil.readVInt(in); int presision = BytesUtil.readVInt(in); infos[i] = new MetricInfo(type, refIndex, presision); } byte[] temp = BytesUtil.readByteArray(in); TableRecordInfoDigest tableInfo = TableRecordInfoDigest.deserialize(temp); return new EndpointAggregators(funcNames, dataTypes, infos, tableInfo); }
public FixedLenMeasureCodec<LongWritable> codec(int col) { return digest.codec(col); }
public int getColumnCount() { return digest.getColumnCount(); }
public static CoprocessorRowType fromTableRecordInfo(TableRecordInfo tableRecordInfo, List<TblColRef> cols) { int[] colSizes = new int[cols.size()]; for (int i = 0; i < cols.size(); i++) { colSizes[i] = tableRecordInfo.getDigest().length(i); } return new CoprocessorRowType(cols.toArray(new TblColRef[cols.size()]), colSizes); }
public RawTableRecord(TableRecordInfoDigest info) { this.digest = info; this.buf = new byte[info.getByteFormLen()]; reset(); }
@Override public TableRecordInfoDigest deserialize(ByteBuffer in) { TableRecordInfoDigest result = new TableRecordInfoDigest(); result.nColumns = BytesUtil.readVInt(in); result.byteFormLen = BytesUtil.readVInt(in); result.offsets = BytesUtil.readIntArray(in); result.dictMaxIds = BytesUtil.readIntArray(in); result.lengths = BytesUtil.readIntArray(in); result.isMetric = BytesUtil.readBooleanArray(in); result.measureSerializers = new FixedLenMeasureCodec<?>[result.nColumns]; for (int i = 0; i < result.nColumns; ++i) { String typeStr = BytesUtil.readAsciiString(in); if (typeStr == null) { result.measureSerializers[i] = null; } else { result.measureSerializers[i] = FixedLenMeasureCodec .get(DataType.getInstance(typeStr)); } } return result; }