private void writeHeader(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.write(MAGIC); // magic header.writeFixed64(rowCount); // row count header.writeFixed32(columnCount); // column count metaData.write(header); // file metadata for (ColumnOutputBuffer column : columns) column.getMeta().write(header); // column metadata for (long start : computeStarts(header.size())) header.writeFixed64(start); // column starts header.writeTo(out); }
ColumnValues(ColumnDescriptor column) throws IOException { this.column = column; this.type = column.metaData.getType(); this.codec = Codec.get(column.metaData); this.checksum = Checksum.get(column.metaData); this.in = new InputBuffer(column.file); column.ensureBlocksRead(); }
@Override public T next() { if (column.metaData.isArray() || column.metaData.getParent() != null) throw new TrevniRuntimeException ("Column is array: " +column.metaData.getName()); try { startRow(); return nextValue(); } catch (IOException e) { throw new TrevniRuntimeException(e); } }
static ColumnMetaData read(InputBuffer in, ColumnFileReader file) throws IOException { ColumnMetaData result = new ColumnMetaData(); MetaData.read(in, result); result.name = result.getString(NAME_KEY); result.type = ValueType.forName(result.getString(TYPE_KEY)); result.values = result.getBoolean(VALUES_KEY); result.isArray = result.getBoolean(ARRAY_KEY); String parentName = result.getString(PARENT_KEY); if (parentName != null) result.setParent(file.getColumnMetaData(parentName)); return result; }
/** Expert: Returns the next length in an array column. */ public int nextLength() throws IOException { if (!column.metaData.isArray()) throw new TrevniRuntimeException ("Column is not array: " +column.metaData.getName()); assert arrayLength == 0; return arrayLength = values.readLength(); }
/** Seek to the named row. */ public void seek(long r) throws IOException { if (r < row || r >= column.lastRow(block)) // not in current block startBlock(column.findBlock(r)); // seek to block start while (r > row && hasNext()) { // skip within block values.skipValue(type); row++; } previous = null; }
static void read(InputBuffer in, MetaData<?> metaData) throws IOException { int size = in.readInt(); for (int i = 0; i < size; i++) metaData.put(in.readString(), in.readBytes()); }
static ColumnFileMetaData read(InputBuffer in) throws IOException { ColumnFileMetaData result = new ColumnFileMetaData(); MetaData.read(in, result); return result; }
public static Codec get(MetaData meta) { String name = meta.getCodec(); if (name == null || "null".equals(name)) return new NullCodec(); else if ("deflate".equals(name)) return new DeflateCodec(); else if ("snappy".equals(name)) return new SnappyCodec(); else if ("bzip2".equals(name)) return new BZip2Codec(); else throw new TrevniRuntimeException("Unknown codec: "+name); }
/** Expert: Must be called before any calls to {@link #nextLength()} or * {@link #nextValue()}. */ public void startRow() throws IOException { if (row >= column.lastRow(block)) { if (block >= column.blockCount()) throw new TrevniRuntimeException("Read past end of column."); startBlock(block+1); } row++; }
/** Set a metadata property to a binary value. */ public T set(String key, byte[] value) { if (isReserved(key)) { throw new TrevniRuntimeException("Cannot set reserved key: " + key); } put(key, value); return (T)this; }
/** Set whether this column has an index of blocks by value. This only makes * sense for sorted columns and permits one to seek into a column by value. */ public ColumnMetaData hasIndexValues(boolean values) { if (isArray) throw new TrevniRuntimeException("Array column cannot have index: "+this); this.values = values; return setReservedBoolean(VALUES_KEY, values); }
public static BlockDescriptor read(InputBuffer in) throws IOException { BlockDescriptor result = new BlockDescriptor(); result.rowCount = in.readFixed32(); result.uncompressedSize = in.readFixed32(); result.compressedSize = in.readFixed32(); return result; }
/** Construct given a name and type. */ public ColumnMetaData(String name, ValueType type) { this.name = name; setReserved(NAME_KEY, name); this.type = type; setReserved(TYPE_KEY, type.getName()); }
/** Write all rows added to the named output stream. */ public void writeTo(OutputStream out) throws IOException { writeHeader(out); for (int column = 0; column < columnCount; column++) columns[column].writeTo(out); }
/** Return an iterator over values in the named column. */ public <T extends Comparable> ColumnValues<T> getValues(String columnName) throws IOException { return new ColumnValues<>(getColumn(columnName)); }
private void flushRun() throws IOException { if (runLength == 0) // not in run return; else if (runLength == 1) // single value getBuffer().writeLength(runValue); else // a run getBuffer().writeLength((3-runValue)-(runLength<<1)); runLength = 0; // reset runValue = NONE; }
/** Set whether this column is an array. */ public ColumnMetaData isArray(boolean isArray) { if (values) throw new TrevniRuntimeException("Array column cannot have index: "+this); this.isArray = isArray; return setReservedBoolean(ARRAY_KEY, isArray); }