@Override public Tuple next() throws IOException { try { if (currentIdx == validIdx) { if (eof) { return null; } else { page(); if(currentIdx == validIdx){ return null; } } } long offset = -1; if(!isCompress()){ offset = fileOffsets.get(currentIdx); } byte[][] cells = BytesUtils.splitPreserveAllTokens(buffer.getData(), startOffsets.get(currentIdx), rowLengthList.get(currentIdx), delimiter, targetColumnIndexes); currentIdx++; return new LazyTuple(schema, cells, offset, nullChars, serde); } catch (Throwable t) { LOG.error("Tuple list length: " + (fileOffsets != null ? fileOffsets.size() : 0), t); LOG.error("Tuple list current index: " + currentIdx, t); throw new IOException(t); } }
Text text = new Text(); reader.getCurrentValue(text); cells = BytesUtils.splitPreserveAllTokens(text.getBytes(), delimiter, projectionMap); totalBytes += (long)text.getBytes().length; tuple = new LazyTuple(schema, cells, 0, nullChars, serde);
int rowKeyFieldIndex = rowKeyFieldIndexes[fieldId]; byte[][] rowKeyFields = BytesUtils.splitPreserveAllTokens( value, rowKeyDelimiter, columnMapping.getNumColumns());
int rowKeyFieldIndex = rowKeyFieldIndexes[fieldId]; byte[][] rowKeyFields = BytesUtils.splitPreserveAllTokens( value, rowKeyDelimiter, columnMapping.getNumColumns());
byte[][] cells = BytesUtils.splitPreserveAllTokens(tuples[currentIdx++], delimiter, targetColumnIndexes); return new LazyTuple(schema, cells, offset); } catch (Throwable t) {
tuples = BytesUtils.splitPreserveAllTokens(buf, rbyte, (char) LF); } else { byte[] lastRow = ArrayUtils.addAll(tail, buf); tuples = BytesUtils.splitPreserveAllTokens(lastRow, rbyte + tail.length, (char) LF); tail = null;
byte[][] rowKeyFields; if (sortSpecs.length > 1) { byte[][] splitValues = BytesUtils.splitPreserveAllTokens( eachEndKey, columnMapping.getRowKeyDelimiter(), columnMapping.getNumColumns()); if (splitValues.length == sortSpecs.length) {
byte[][] rowKeyFields; if (sortSpecs.length > 1) { byte[][] splitValues = BytesUtils.splitPreserveAllTokens( eachEndKey, columnMapping.getRowKeyDelimiter(), columnMapping.getNumColumns()); if (splitValues.length == sortSpecs.length) {
sb.append(new String(nullbytes)).append('|'); sb.append(NullDatum.get()); textRow = BytesUtils.splitPreserveAllTokens(sb.toString().getBytes(), '|', 13); serde = new TextSerializerDeserializer(); serde.init(schema);
@Test public void testInvalidNumber() { byte[][] bytes = BytesUtils.splitPreserveAllTokens(" 1| |2 ||".getBytes(), '|', 5); Schema schema = SchemaBuilder.builder() .add("col1", TajoDataTypes.Type.INT2) .add("col2", TajoDataTypes.Type.INT4) .add("col3", TajoDataTypes.Type.INT8) .add("col4", TajoDataTypes.Type.FLOAT4) .add("col5", TajoDataTypes.Type.FLOAT8) .build(); LazyTuple tuple = new LazyTuple(schema, bytes, 0); assertEquals(bytes.length, tuple.size()); for (int i = 0; i < tuple.size(); i++){ assertEquals(NullDatum.get(), tuple.get(i)); } }