public Slice append(TableRecord rec) { short shard = rec.getShard(); return builders[shard].append(rec); }
public List<Slice> close() { List<Slice> result = Lists.newArrayList(); for (SliceBuilder builder : builders) { Slice slice = builder.close(); if (slice != null) result.add(slice); } return result; }
public Slice append(TableRecord rec) { if (rec.getShard() != shard) throw new IllegalStateException(); Slice doneSlice = null; if (isFull()) { doneSlice = doneSlice(); } if (nRecords == 0) { sliceTimestamp = increaseSliceTimestamp(rec.getTimestamp()); } nRecords++; for (int i = 0; i < nColumns; i++) { rec.getValueBytes(i, temp); containers[i].append(temp); } return doneSlice; }
@Override public void reduce(LongWritable key, Iterable<ImmutableBytesWritable> values, Context context) // throws IOException, InterruptedException { for (ImmutableBytesWritable v : values) { rec.setBytes(v.get(), v.getOffset(), v.getLength()); if (builder == null) { builder = new SliceBuilder(info, rec.getShard()); } //TODO: to delete this log System.out.println(rec.getShard() + " - " + rec); Slice slice = builder.append(rec); if (slice != null) { output(slice, context); } } }
public ShardingSliceBuilder(TableRecordInfo info) { int sharding = info.getDescriptor().getSharding(); builders = new SliceBuilder[sharding]; for (short i = 0; i < sharding; i++) { builders[i] = new SliceBuilder(info, i); } }
public Slice close() { Slice doneSlice = doneSlice(); this.sliceTimestamp = Long.MIN_VALUE; this.nRecords = 0; return doneSlice; }
public Slice buildSlice(StreamingBatch microStreamBatch) throws IOException{ final List<List<String>> messages = Lists.transform(microStreamBatch.getMessages(), new Function<StreamingMessage, List<String>>() { @Nullable @Override public List<String> apply(@Nullable StreamingMessage input) { return input.getData(); } }); final Dictionary<?>[] dictionaries = IIDictionaryBuilder.buildDictionary(messages, iiDesc); TableRecordInfo tableRecordInfo = new TableRecordInfo(iiDesc, dictionaries); return build(messages, tableRecordInfo, dictionaries); }
@Override public void reduce(LongWritable key, Iterable<ImmutableBytesWritable> values, Context context) // throws IOException, InterruptedException { for (ImmutableBytesWritable v : values) { rec.setBytes(v.get(), v.getOffset(), v.getLength()); if (builder == null) { builder = new SliceBuilder(info, rec.getShard()); } //TODO: to delete this log System.out.println(rec.getShard() + " - " + rec); Slice slice = builder.append(rec); if (slice != null) { output(slice, context); } } }
public SliceBuilder(TableRecordInfo info, short shard) { this.info = info; this.nColumns = info.getDigest().getColumnCount(); this.nRecordsCap = Math.max(1, info.getDescriptor().getSliceSize()); this.shard = shard; this.sliceTimestamp = Long.MIN_VALUE; this.nRecords = 0; this.containers = null; doneSlice(); // init containers }
@Override protected void cleanup(Context context) throws IOException, InterruptedException { Slice slice = builder.close(); if (slice != null) { output(slice, context); } }
@Override protected void cleanup(Context context) throws IOException, InterruptedException { Slice slice = builder.close(); if (slice != null) { output(slice, context); } }