int size = decodeVIntSize((byte) len);
public static int[][] split(byte[] str, int offset, int length, byte[] separator, int[][] indices) { if (indices.length == 0) { return indices; // trivial } final int limit = offset + length; int start = offset; int colIndex = 0; for (int index = offset; index < limit;) { if (onDelimiter(str, index, limit, separator)) { indices[colIndex++] = new int[] {start, index}; if (colIndex >= indices.length) { return indices; } index += separator.length; start = index; } else { index++; } } if (colIndex < indices.length) { indices[colIndex] = new int[]{start, limit}; } return indices; }
private static byte[][] splitWorker(byte[] str, int offset, int length, char separatorChar, int[] target, int numColumns) { return splitWorker(str, offset, length, new byte[] {(byte)separatorChar}, target, numColumns); }
@Override public Tuple next() throws IOException { try { if (currentIdx == validIdx) { if (eof) { return null; } else { page(); if(currentIdx == validIdx){ return null; } } } long offset = -1; if(!isCompress()){ offset = fileOffsets.get(currentIdx); } byte[][] cells = BytesUtils.splitPreserveAllTokens(buffer.getData(), startOffsets.get(currentIdx), rowLengthList.get(currentIdx), delimiter, targetColumnIndexes); currentIdx++; return new LazyTuple(schema, cells, offset, nullChars, serde); } catch (Throwable t) { LOG.error("Tuple list length: " + (fileOffsets != null ? fileOffsets.size() : 0), t); LOG.error("Tuple list current index: " + currentIdx, t); throw new IOException(t); } }
public static byte[] toAsciiBytes(long i) { return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray()); }
BytesUtils.writeVLong(os, tuple.getTextBytes(j).length); break; case PROTOBUF: ProtobufDatum protobufDatum = (ProtobufDatum) tuple.getProtobufDatum(j); BytesUtils.writeVLong(os, protobufDatum.asByteArray().length); break; case CHAR: case BLOB: BytesUtils.writeVLong(os, tuple.getBytes(j).length); break; default:
byte [][] padded = BytesUtils.padBytes(a, b); a = padded[0]; b = padded[1];
int[][] indices = split(str, offset, length, separator, new int[indexMax][]); byte[][] result = new byte[numColumns][];
mappingColumns[index] = new byte[2][]; byte[][] mappingTokens = BytesUtils.splitTrivial(eachToken.trim().getBytes(), (byte)':');
Text text = new Text(); reader.getCurrentValue(text); cells = BytesUtils.splitPreserveAllTokens(text.getBytes(), delimiter, projectionMap); totalBytes += (long)text.getBytes().length; tuple = new LazyTuple(schema, cells, 0, nullChars, serde);
public static byte[] toAsciiBytes(Number i) { return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray()); }
BytesUtils.writeVLong(os, tuple.getTextBytes(j).length); break; case PROTOBUF: ProtobufDatum protobufDatum = (ProtobufDatum) tuple.getProtobufDatum(j); BytesUtils.writeVLong(os, protobufDatum.asByteArray().length); break; case CHAR: case INET4: case BLOB: BytesUtils.writeVLong(os, tuple.getBytes(j).length); break; default:
byte [][] padded = BytesUtils.padBytes(a, b); a = padded[0]; b = padded[1];
int[][] indices = split(str, offset, length, separator, new int[indexMax][]); byte[][] result = new byte[numColumns][];
mappingColumns[index] = new byte[2][]; byte[][] mappingTokens = BytesUtils.splitTrivial(eachToken.trim().getBytes(), (byte)':');
int rowKeyFieldIndex = rowKeyFieldIndexes[fieldId]; byte[][] rowKeyFields = BytesUtils.splitPreserveAllTokens( value, rowKeyDelimiter, columnMapping.getNumColumns());
public static byte[] toAsciiBytes(double i) { return BytesUtils.toASCIIBytes(String.valueOf(i).toCharArray()); }
BytesUtils.writeVLong(os, datum.asTextBytes().length); break; case PROTOBUF: ProtobufDatum protobufDatum = (ProtobufDatum) datum; BytesUtils.writeVLong(os, protobufDatum.asByteArray().length); break; case CHAR: case INET4: case BLOB: BytesUtils.writeVLong(os, datum.asByteArray().length); break; default:
private static byte[][] splitWorker(byte[] str, int offset, int length, char separatorChar, int[] target, int numColumns) { return splitWorker(str, offset, length, new byte[] {(byte)separatorChar}, target, numColumns); }
byte[][] padded = BytesUtils.padBytes(startBytes, endBytes); range.getStart().put(i, DatumFactory.createText(padded[0])); range.getEnd().put(i, DatumFactory.createText(padded[1]));