Refine search
protected void outputKV(String[] flatRow, Context context) throws IOException, InterruptedException { byte[] rowKey = baseCuboidBuilder.buildKey(flatRow); outputKey.set(rowKey, 0, rowKey.length); ByteBuffer valueBuf = baseCuboidBuilder.buildValue(flatRow); outputValue.set(valueBuf.array(), 0, valueBuf.position()); context.write(outputKey, outputValue); }
public void map(Object key, Text value, Context context) throws IOException, InterruptedException { StringTokenizer tokens = new StringTokenizer(value.toString(), " \t\n\r\f%"); String attr = tokens.nextToken(); if (attr.endsWith(":tput_samples")){ String[] tags=attr.split(":"); String[] samples = tokens.nextToken().split(";"); for(int j=0; !samples[j].startsWith("EoR"); j++){ t.set(samples[j]); context.write(new Text(tags[1]), t); } } } }
@Override protected void doCleanup(Context context) throws IOException, InterruptedException { ByteBuffer hllBuf = ByteBuffer.allocate(BufferedMeasureCodec.DEFAULT_BUFFER_SIZE); HLLCounter hll; for (int i = 0; i < cuboidIds.length; i++) { hll = allCuboidsHLL[i]; outputKey.set(Bytes.toBytes(cuboidIds[i])); hllBuf.clear(); hll.writeRegisters(hllBuf); outputValue.set(hllBuf.array(), 0, hllBuf.position()); context.write(outputKey, outputValue); } } }
@Override public void doMap(NullWritable key, Text value, Context context) throws IOException, InterruptedException { tmpBuf.clear(); int size = value.getLength()+ 1; if (size >= tmpBuf.capacity()) { tmpBuf = ByteBuffer.allocate(countNewSize(tmpBuf.capacity(), size)); } tmpBuf.put(Bytes.toBytes(index)[3]); tmpBuf.put(value.getBytes(), 0, value.getLength()); outputKey.set(tmpBuf.array(), 0, tmpBuf.position()); sortableKey.init(outputKey, type); context.write(sortableKey, NullWritable.get()); }
private void writeFieldValue(Context context, DataType type, Integer colIndex, String value) throws IOException, InterruptedException { int reducerIndex = reducerMapping.getReducerIdForCol(colIndex, value); tmpbuf.clear(); byte[] valueBytes = Bytes.toBytes(value); int size = valueBytes.length + 1; if (size >= tmpbuf.capacity()) { tmpbuf = ByteBuffer.allocate(countNewSize(tmpbuf.capacity(), size)); } tmpbuf.put(Bytes.toBytes(reducerIndex)[3]); tmpbuf.put(valueBytes); outputKey.set(tmpbuf.array(), 0, tmpbuf.position()); sortableKey.init(outputKey, type); context.write(sortableKey, EMPTY_TEXT); // log a few rows for troubleshooting if (rowCount < 10) { logger.info("Sample output: " + allCols.get(colIndex) + " '" + value + "' => reducer " + reducerIndex); } }
static Text nextString(ColumnVector vector, int row, Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { Text result; if (previous == null || previous.getClass() != Text.class) { result = new Text(); } else { result = (Text) previous; } BytesColumnVector bytes = (BytesColumnVector) vector; result.set(bytes.vector[row], bytes.start[row], bytes.length[row]); return result; } else { return null; } }
@Override public void doMap(LongWritable key, BytesWritable value, Context context) throws IOException, InterruptedException { ByteBuffer buffer = ByteBuffer.wrap(value.getBytes(), 0, value.getLength()); StreamingMessageRow row = streamingParser.parse(buffer).get(0); if (row == null) { throw new IllegalArgumentException(""); } data = StringUtil.join(row.getData(), delimiter); // output this row to value outValue.set(Bytes.toBytes(data)); context.write(outKey, outValue); } }
public static Text nextString(ColumnVector vector, int row, Object previous) { if (vector.isRepeating) { row = 0; } if (vector.noNulls || !vector.isNull[row]) { Text result; if (previous == null || previous.getClass() != Text.class) { result = new Text(); } else { result = (Text) previous; } BytesColumnVector bytes = (BytesColumnVector) vector; result.set(bytes.vector[row], bytes.start[row], bytes.length[row]); return result; } else { return null; } }
tmpbuf.put((byte) FactDistinctColumnsReducerMapping.MARK_FOR_HLL_COUNTER); // one byte tmpbuf.putLong(cuboidIds[i]); outputKey.set(tmpbuf.array(), 0, tmpbuf.position()); hllBuf.clear(); hll.writeRegisters(hllBuf); outputValue.set(hllBuf.array(), 0, hllBuf.position()); sortableKey.init(outputKey, (byte) 0); context.write(sortableKey, outputValue);