public ColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { this.writer = writer; this.meta = meta; this.codec = Codec.get(meta); this.checksum = Checksum.get(meta); this.buffer = new OutputBuffer(); this.blockDescriptors = new ArrayList<>(); this.blockData = new ArrayList<>(); if (meta.hasIndexValues()) this.firstValues = new ArrayList<>(); }
public void writeTo(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.writeFixed32(blockDescriptors.size()); for (int i = 0; i < blockDescriptors.size(); i++) { blockDescriptors.get(i).writeTo(header); if (meta.hasIndexValues()) header.write(firstValues.get(i)); } header.writeTo(out); for (byte[] data : blockData) out.write(data); }
public void writeValue(Object value) throws IOException { buffer.writeValue(value, meta.getType()); if (meta.hasIndexValues() && rowCount == 0) firstValues.add(buffer.toByteArray()); }
public ArrayColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { super(writer, meta); assert getMeta().isArray() || getMeta().getParent() != null; assert !getMeta().hasIndexValues(); }
/** Seek to the named value. */ public void seek(T v) throws IOException { if (!column.metaData.hasIndexValues()) throw new TrevniRuntimeException ("Column does not have value index: " +column.metaData.getName()); if (previous == null // not in current block? || previous.compareTo(v) > 0 || (block != column.blockCount()-1 && column.firstValues[block+1].compareTo(v) <= 0)) startBlock(column.findBlock(v)); // seek to block start while (hasNext()) { // scan block long savedPosition = values.tell(); T savedPrevious = previous; if (next().compareTo(v) >= 0) { values.seek(savedPosition); previous = savedPrevious; row--; return; } } }
public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }
new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.STRING) .hasIndexValues(true));
public ColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { this.writer = writer; this.meta = meta; this.codec = Codec.get(meta); this.checksum = Checksum.get(meta); this.buffer = new OutputBuffer(); this.blockDescriptors = new ArrayList<BlockDescriptor>(); this.blockData = new ArrayList<byte[]>(); if (meta.hasIndexValues()) this.firstValues = new ArrayList<byte[]>(); }
public void writeTo(OutputStream out) throws IOException { OutputBuffer header = new OutputBuffer(); header.writeFixed32(blockDescriptors.size()); for (int i = 0; i < blockDescriptors.size(); i++) { blockDescriptors.get(i).writeTo(header); if (meta.hasIndexValues()) header.write(firstValues.get(i)); } header.writeTo(out); for (byte[] data : blockData) out.write(data); }
public void writeValue(Object value) throws IOException { buffer.writeValue(value, meta.getType()); if (meta.hasIndexValues() && rowCount == 0) firstValues.add(buffer.toByteArray()); }
public ArrayColumnOutputBuffer(ColumnFileWriter writer, ColumnMetaData meta) throws IOException { super(writer, meta); assert getMeta().isArray() || getMeta().getParent() != null; assert !getMeta().hasIndexValues(); }
/** Seek to the named value. */ public void seek(T v) throws IOException { if (!column.metaData.hasIndexValues()) throw new TrevniRuntimeException ("Column does not have value index: " +column.metaData.getName()); if (previous == null // not in current block? || previous.compareTo(v) > 0 || (block != column.blockCount()-1 && column.firstValues[block+1].compareTo(v) <= 0)) startBlock(column.findBlock(v)); // seek to block start while (hasNext()) { // scan block long savedPosition = values.tell(); T savedPrevious = previous; if (next().compareTo(v) >= 0) { values.seek(savedPosition); previous = savedPrevious; row--; return; } } }
public void ensureBlocksRead() throws IOException { if (blocks != null) return; // read block descriptors InputBuffer in = new InputBuffer(file, start); int blockCount = in.readFixed32(); BlockDescriptor[] blocks = new BlockDescriptor[blockCount]; if (metaData.hasIndexValues()) firstValues = (T[])new Comparable[blockCount]; for (int i = 0; i < blockCount; i++) { blocks[i] = BlockDescriptor.read(in); if (metaData.hasIndexValues()) firstValues[i] = in.<T>readValue(metaData.getType()); } dataStart = in.tell(); // compute blockStarts and firstRows Checksum checksum = Checksum.get(metaData); blockStarts = new long[blocks.length]; firstRows = new long[blocks.length]; long startPosition = dataStart; long row = 0; for (int i = 0; i < blockCount; i++) { BlockDescriptor b = blocks[i]; blockStarts[i] = startPosition; firstRows[i] = row; startPosition += b.compressedSize + checksum.size(); row += b.rowCount; } this.blocks = blocks; }
new ColumnFileWriter(createFileMeta(), new ColumnMetaData("test", ValueType.STRING) .hasIndexValues(true));