/** * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes. * * @param input The byte[] to increment. * @return The incremented copy of "in". May be same length or 1 byte longer. */ public static byte[] unsignedCopyAndIncrement(final byte[] input) { byte[] copy = copy(input); if (copy == null) { throw new IllegalArgumentException("cannot increment null array"); } for (int i = copy.length - 1; i >= 0; --i) { if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum copy[i] = 0; } else { ++copy[i]; return copy; } } // we maxed out the array byte[] out = new byte[copy.length + 1]; out[0] = 1; System.arraycopy(copy, 0, out, 1, copy.length); return out; }
public byte[] toBytes() { return Bytes.copy(this.array(), this.offset(), this.length()); }
@Override public byte[] convertToBytes(String v) { v = normalizeNumber(v); NumberBytesCodec codec = getCodec(this.maxDigitsBeforeDecimalPoint); byte[] num = Bytes.toBytes(v); codec.encodeNumber(num, 0, num.length); return Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen); }
private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) { int partationId = 0; ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>(); for (String str : strNumList) { ByteBuffer keyBuffer = ByteBuffer.allocate(4096); int offset = keyBuffer.position(); keyBuffer.put(Bytes.toBytes(partationId)[3]); keyBuffer.put(Bytes.toBytes(str)); Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1); Text outputKey = new Text(); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(); sortableKey.init(outputKey, typeFlag); keyList.add(sortableKey); } return keyList; }
private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList, byte typeFlag) { int partationId = 0; ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>(); for (String str : strNumList) { ByteBuffer keyBuffer = ByteBuffer.allocate(4096); int offset = keyBuffer.position(); keyBuffer.put(Bytes.toBytes(partationId)[3]); keyBuffer.put(Bytes.toBytes(str)); //System.out.println(Arrays.toString(keyBuffer.array())); byte[] valueField = Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1); //System.out.println("new string:"+new String(valueField)); //System.out.println("arrays toString:"+Arrays.toString(valueField)); Text outputKey = new Text(); outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset); SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(); sortableKey.init(outputKey, typeFlag); keyList.add(sortableKey); } return keyList; }
builder.addValue(value); } else { byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
builder.addValue(value); } else { byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
public RawTableRecord(RawTableRecord another) { this.digest = another.digest; this.buf = Bytes.copy(another.buf); }
/** * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes. * * @param input The byte[] to increment. * @return The incremented copy of "in". May be same length or 1 byte longer. */ public static byte[] unsignedCopyAndIncrement(final byte[] input) { byte[] copy = copy(input); if (copy == null) { throw new IllegalArgumentException("cannot increment null array"); } for (int i = copy.length - 1; i >= 0; --i) { if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum copy[i] = 0; } else { ++copy[i]; return copy; } } // we maxed out the array byte[] out = new byte[copy.length + 1]; out[0] = 1; System.arraycopy(copy, 0, out, 1, copy.length); return out; }
@Override public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { HashSet<ByteArray> set = new HashSet<ByteArray>(); for (Text textValue : values) { ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength())); set.add(value); } for (ByteArray value : set) { outputValue.set(value.data); context.write(key, outputValue); } }
@Override public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { HashSet<ByteArray> set = new HashSet<ByteArray>(); for (Text textValue : values) { ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength())); set.add(value); } for (ByteArray value : set) { outputValue.set(value.data); context.write(key, outputValue); } }
/** * Treat the byte[] as an unsigned series of bytes, most significant bits first. Start by adding * 1 to the rightmost bit/byte and carry over all overflows to the more significant bits/bytes. * * @param input The byte[] to increment. * @return The incremented copy of "in". May be same length or 1 byte longer. */ public static byte[] unsignedCopyAndIncrement(final byte[] input) { byte[] copy = copy(input); if (copy == null) { throw new IllegalArgumentException("cannot increment null array"); } for (int i = copy.length - 1; i >= 0; --i) { if (copy[i] == -1) {// -1 is all 1-bits, which is the unsigned maximum copy[i] = 0; } else { ++copy[i]; return copy; } } // we maxed out the array byte[] out = new byte[copy.length + 1]; out[0] = 1; System.arraycopy(copy, 0, out, 1, copy.length); return out; }
@Override public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { String columnName = columns[key.get()]; HashSet<ByteArray> set = new HashSet<ByteArray>(); for (Text textValue : values) { ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength())); set.add(value); } Configuration conf = context.getConfiguration(); FileSystem fs = FileSystem.get(conf); String outputPath = conf.get(BatchConstants.OUTPUT_PATH); FSDataOutputStream out = fs.create(new Path(outputPath, columnName)); try { for (ByteArray value : set) { out.write(value.data); out.write('\n'); } } finally { out.close(); } }
@Override public void addValue(byte[] value) { codec.encodeNumber(value, 0, value.length); byte[] copy = Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen); super.addValue(copy); }
@Override public void reduce(ShortWritable key, Iterable<Text> values, Context context) throws IOException, InterruptedException { TblColRef col = columnList.get(key.get()); HashSet<ByteArray> set = new HashSet<ByteArray>(); for (Text textValue : values) { ByteArray value = new ByteArray(Bytes.copy(textValue.getBytes(), 0, textValue.getLength())); set.add(value); } Configuration conf = context.getConfiguration(); FileSystem fs = FileSystem.get(conf); String outputPath = conf.get(BatchConstants.OUTPUT_PATH); FSDataOutputStream out = fs.create(new Path(outputPath, col.getName())); try { for (ByteArray value : set) { out.write(value.data); out.write('\n'); } } finally { out.close(); } }
@Override public boolean moveNext() throws IOException { if (curDictIndex < dictionaryList.size() && curKey <= curDict.getMaxId()) { byte[] buffer = new byte[curDict.getSizeOfValue()]; int size = curDict.getValueBytesFromId(curKey, buffer, 0); curValue = Bytes.copy(buffer, 0, size); if (++curKey > curDict.getMaxId()) { if (++curDictIndex < dictionaryList.size()) { curDict = dictionaryList.get(curDictIndex); curKey = curDict.getMinId(); } } return true; } curValue = null; return false; }
public byte[] toBytes() { return Bytes.copy(this.array(), this.offset(), this.length()); }
@Override public byte[] convertToBytes(String v) { v = normalizeNumber(v); NumberBytesCodec codec = getCodec(this.maxDigitsBeforeDecimalPoint); byte[] num = Bytes.toBytes(v); codec.encodeNumber(num, 0, num.length); return Bytes.copy(codec.buf, codec.bufOffset, codec.bufLen); }
builder.addValue(value); } else { byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);
builder.addValue(value); } else { byte[] keyBytes = Bytes.copy(key.getBytes(), 1, key.getLength() - 1);