@Override public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException { rec.reset(); for (int i = 0; i < fields.size(); i++) { Object fieldValue = record.get(i); rec.setValueString(i, fieldValue == null? null : fieldValue.toString()); } outputKey.set(rec.getTimestamp()); // outputValue's backing bytes array is the same as rec context.write(outputKey, outputValue); } }
@Override public Object clone() { return new TableRecord(this); }
public Slice append(TableRecord rec) { if (rec.getShard() != shard) throw new IllegalStateException(); Slice doneSlice = null; if (isFull()) { doneSlice = doneSlice(); } if (nRecords == 0) { sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp()); } nRecords++; for (int i = 0; i < nColumns; i++) { rec.getValueBytes(i, temp); containers[i].append(temp); } return doneSlice; }
public static void main(String[] args) throws IOException { Configuration hconf = HadoopUtil.getCurrentConfiguration(); IIManager mgr = IIManager.getInstance(KylinConfig.getInstanceFromEnv()); String iiName = args[0]; IIInstance ii = mgr.getII(iiName); String path = args[1]; System.out.println("Reading from " + path + " ..."); TableRecordInfo info = new TableRecordInfo(ii.getFirstSegment()); IIKeyValueCodec codec = new IIKeyValueCodec(info.getDigest()); int count = 0; for (Slice slice : codec.decodeKeyValue(readSequenceKVs(hconf, path))) { for (RawTableRecord rec : slice) { System.out.printf(new TableRecord(rec, info).toString()); count++; } } System.out.println("Total " + count + " records"); }
temp1.setValueString(factTableColumnIndex[0], "10000000239"); temp1.setValueString(factTableColumnIndex[1], "2012-03-22"); temp1.setValueString(factTableColumnIndex[2], "Auction"); temp1.setValueString(factTableColumnIndex[3], "80135"); temp1.setValueString(factTableColumnIndex[4], "0"); temp1.setValueString(factTableColumnIndex[5], "14"); temp1.setValueString(factTableColumnIndex[6], "199.99"); temp1.setValueString(factTableColumnIndex[7], "1"); temp1.setValueString(factTableColumnIndex[8], "10000005"); temp2.setValueString(factTableColumnIndex[0], "10000000244"); temp2.setValueString(factTableColumnIndex[1], "2012-11-11"); temp2.setValueString(factTableColumnIndex[2], "Auction"); temp2.setValueString(factTableColumnIndex[3], "16509"); temp2.setValueString(factTableColumnIndex[4], "101"); temp2.setValueString(factTableColumnIndex[5], "12"); temp2.setValueString(factTableColumnIndex[6], "2.09"); temp2.setValueString(factTableColumnIndex[7], "1"); temp2.setValueString(factTableColumnIndex[8], "10000004"); temp3.setValueString(factTableColumnIndex[0], "10000000259"); temp3.setValueString(factTableColumnIndex[1], "2012-07-12"); temp3.setValueString(factTableColumnIndex[2], "Others"); temp3.setValueString(factTableColumnIndex[3], "15687"); temp3.setValueString(factTableColumnIndex[4], "0"); temp3.setValueString(factTableColumnIndex[5], "14"); temp3.setValueString(factTableColumnIndex[6], "100"); temp3.setValueString(factTableColumnIndex[7], "1"); temp3.setValueString(factTableColumnIndex[8], "10000020");
@Override public int compare(TableRecord a, TableRecord b) { long x = a.getTimestamp() - b.getTimestamp(); if (x > 0) return 1; else if (x == 0) return 0; else return -1; } });
public Slice makeSlice(TableRecordInfoDigest digest, List<TableRecord> records) { Preconditions.checkArgument(records != null && !records.isEmpty(), "records cannot be empty"); Preconditions.checkArgument(records.size() <= nRecordsCap, "batch count cannot exceed " + nRecordsCap); sliceTimestamp = increaseSliceTimestamp(records.get(0).getTimestamp()); ColumnValueContainer[] containers = new ColumnValueContainer[nColumns]; for (int i : desc.getValueColumns()) { containers[i] = new CompressedValueContainer(digest, i, nRecordsCap); } for (int i : desc.getMetricsColumns()) { containers[i] = new CompressedValueContainer(digest, i, nRecordsCap); } for (TableRecord record : records) { for (int i = 0; i < nColumns; i++) { record.getValueBytes(i, temp); containers[i].append(temp); } } return new Slice(digest, shard, sliceTimestamp, containers); }
private void dump(Iterable<TableRecord> records) { for (TableRecord rec : records) { System.out.println(rec.toString()); } }
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME)); IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW); this.info = new TableRecordInfo(seg); this.rec = this.info.createTableRecord(); outputKey = new LongWritable(); outputValue = new ImmutableBytesWritable(rec.getBytes()); schema = HCatInputFormat.getTableSchema(context.getConfiguration()); fields = schema.getFields(); }
public Slice append(TableRecord rec) { short shard = rec.getShard(); return builders[shard].append(rec); }
@Override public ITuple next() { if (!hasNext()) { throw new IllegalStateException("No more Tuple in the SingleRegionTupleIterator"); } IIProtos.IIResponse.IIRow currentRow = rows.get(index); byte[] columnsBytes = currentRow.getColumns().toByteArray(); this.tableRecord.setBytes(columnsBytes, 0, columnsBytes.length); if (currentRow.hasMeasures()) { byte[] measuresBytes = currentRow.getMeasures().toByteArray(); this.measureValues = pushedDownAggregators.deserializeMetricValues(measuresBytes, 0); } index++; return makeTuple(this.tableRecord, this.measureValues); }
public Slice append(TableRecord rec) { if (rec.getShard() != shard) throw new IllegalStateException(); Slice doneSlice = null; if (isFull()) { doneSlice = doneSlice(); } if (nRecords == 0) { sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp()); } nRecords++; for (int i = 0; i < nColumns; i++) { rec.getValueBytes(i, temp); containers[i].append(temp); } return doneSlice; }
@Override public void reduce(LongWritable key, Iterable<ImmutableBytesWritable> values, Context context) // throws IOException, InterruptedException { for (ImmutableBytesWritable v : values) { rec.setBytes(v.get(), v.getOffset(), v.getLength()); if (builder == null) { builder = new SliceBuilder(info, rec.getShard()); } //TODO: to delete this log System.out.println(rec.getShard() + " - " + rec); Slice slice = builder.append(rec); if (slice != null) { output(slice, context); } } }
private List<TableRecord> loadRecordsSorted() throws IOException { File file = new File(LOCALMETA_TEST_DATA, "data/flatten_data_for_ii.csv"); FileInputStream in = new FileInputStream(file); List<String> lines = IOUtils.readLines(in, "UTF-8"); in.close(); List<TableRecord> records = Lists.newArrayList(); for (String line : lines) { String[] fields = line.split(","); TableRecord rec = info.createTableRecord(); for (int col = 0; col < fields.length; col++) { rec.setValueString(col, fields[col]); } records.add(rec); } Collections.sort(records, new Comparator<TableRecord>() { @Override public int compare(TableRecord a, TableRecord b) { long x = a.getTimestamp() - b.getTimestamp(); if (x > 0) return 1; else if (x == 0) return 0; else return -1; } }); return records; }
@Test @SuppressWarnings("rawtypes") public void basicTest() { for (int i = 0; i < tableData.size(); ++i) { byte[] data = tableData.get(i).getBytes(); CoprocessorProjector.AggrKey aggKey = projector.getAggrKey(data); MeasureAggregator[] bufs = aggCache.getBuffer(aggKey); aggregators.aggregate(bufs, data); aggCache.checkMemoryUsage(); } assertEquals(aggCache.getAllEntries().size(), 2); long sumTotal = 0; long minTotal = 0; for (Map.Entry<CoprocessorProjector.AggrKey, MeasureAggregator[]> entry : aggCache.getAllEntries()) { sumTotal += ((LongWritable) entry.getValue()[0].getState()).get(); minTotal += ((LongWritable) entry.getValue()[1].getState()).get(); } assertEquals(3020800, sumTotal); assertEquals(1020900, minTotal); }
public Slice append(TableRecord rec) { short shard = rec.getShard(); return builders[shard].append(rec); }
@Override public void map(KEYIN key, HCatRecord record, Context context) throws IOException, InterruptedException { rec.reset(); for (int i = 0; i < fields.size(); i++) { Object fieldValue = record.get(i); rec.setValueString(i, fieldValue == null ? null : fieldValue.toString()); } outputKey.set(rec.getTimestamp()); // outputValue's backing bytes array is the same as rec context.write(outputKey, outputValue); } }