public static void write(File file, Iterator<DenseMatrixRow> rows) throws IOException { write(file, rows, new ValueConf()); } public static void write(File file, Iterator<DenseMatrixRow> rows, ValueConf vconf) throws IOException {
public SparseMatrixRow(ValueConf vconf, int rowIndex, int colIds[], float colVals[]) { this.vconf = vconf; short packed[] = new short[colVals.length]; for (int i = 0; i < colVals.length; i++) { packed[i] = vconf.pack(colVals[i]); } createBuffer(rowIndex, colIds, packed); }
public final short pack(float s) { float normalized = (pinchScore(s) - minScore) / range; return (short)(normalized * PACKED_RANGE + Short.MIN_VALUE); }
@Override public final float getColValue(int i) { return vconf.unpack(valBuffer.get(i)); }
public synchronized void writeRow(SparseMatrixRow row) throws IOException { if (!row.getValueConf().almostEquals(vconf)) { throw new IllegalArgumentException("Value conf for row does not match the writer's value conf"); } row.getBuffer().rewind(); byte[] bytes = new byte[row.getBuffer().remaining()]; row.getBuffer().get(bytes, 0, bytes.length); rowOffsets.put(row.getRowIndex(), bodyOffset); rowIndexes.add(row.getRowIndex()); body.write(bytes); bodyOffset += bytes.length; // pad rows to 8 byte offsets to speed things up. while (bodyOffset % 8 != 0) { bodyOffset++; body.write(ROW_PADDING); } }
@Override public final float getColValue(int i) { return vconf.unpack(valBuffer.get(i)); }
public synchronized void writeRow(SparseMatrixRow row) throws IOException { if (!row.getValueConf().almostEquals(vconf)) { throw new IllegalArgumentException("Value conf for row does not match the writer's value conf"); } row.getBuffer().rewind(); byte[] bytes = new byte[row.getBuffer().remaining()]; row.getBuffer().get(bytes, 0, bytes.length); rowOffsets.put(row.getRowIndex(), bodyOffset); rowIndexes.add(row.getRowIndex()); body.write(bytes); bodyOffset += bytes.length; // pad rows to 8 byte offsets to speed things up. while (bodyOffset % 8 != 0) { bodyOffset++; body.write(ROW_PADDING); } }
public static void write(File file, Iterator<SparseMatrixRow> rows) throws IOException { write(file, rows, new ValueConf()); } public static void write(File file, Iterator<SparseMatrixRow> rows, ValueConf vconf) throws IOException {
public SparseMatrixRow(ValueConf vconf, int rowIndex, int colIds[], float colVals[]) { this.vconf = vconf; short packed[] = new short[colVals.length]; for (int i = 0; i < colVals.length; i++) { packed[i] = vconf.pack(colVals[i]); } createBuffer(rowIndex, colIds, packed); }
@Override public final float getColValue(int i) { return vconf.unpack(valBuffer.get(i)); }
public synchronized void writeRow(DenseMatrixRow row) throws IOException { if (!row.getValueConf().almostEquals(vconf)) { throw new IllegalArgumentException("Value conf for row does not match the writer's value conf"); } if (colIds == null) { colIds = row.getColIds(); } if (!Arrays.equals(colIds, row.getColIds())) { throw new IllegalArgumentException("Column id mismatch for row " + row.getRowIndex()); } row.getBuffer().rewind(); byte[] bytes = new byte[row.getBuffer().remaining()]; row.getBuffer().get(bytes, 0, bytes.length); rowOffsets.put(row.getRowIndex(), bodyOffset); rowIndexes.add(row.getRowIndex()); body.write(bytes); bodyOffset += bytes.length; // pad rows to 8 byte offsets to speed things up. while (bodyOffset % 8 != 0) { bodyOffset++; body.write(ROW_PADDING); } }
public final short pack(float s) { float normalized = (pinchScore(s) - minScore) / range; return (short)(normalized * PACKED_RANGE + Short.MIN_VALUE); }
public static void write(File file, Iterator<DenseMatrixRow> rows) throws IOException { write(file, rows, new ValueConf()); } public static void write(File file, Iterator<DenseMatrixRow> rows, ValueConf vconf) throws IOException {
public DenseMatrixRow(ValueConf vconf, int rowIndex, int colIds[], float colVals[]) { if (!SparseMatrixUtils.isIncreasing(colIds)) { throw new IllegalArgumentException("Columns must be sorted by id"); } this.vconf = vconf; this.c1 = vconf.c1; this.c2 = vconf.c2; this.colIds = colIds; short packed[] = new short[colVals.length]; for (int i = 0; i < colVals.length; i++) { packed[i] = vconf.pack(colVals[i]); } createBuffer(rowIndex, colIds, packed); }
@Override public final float getColValue(int i) { return vconf.unpack(valBuffer.get(i)); }
public synchronized void writeRow(DenseMatrixRow row) throws IOException { if (!row.getValueConf().almostEquals(vconf)) { throw new IllegalArgumentException("Value conf for row does not match the writer's value conf"); } if (colIds == null) { colIds = row.getColIds(); } if (!Arrays.equals(colIds, row.getColIds())) { throw new IllegalArgumentException("Column id mismatch for row " + row.getRowIndex()); } row.getBuffer().rewind(); byte[] bytes = new byte[row.getBuffer().remaining()]; row.getBuffer().get(bytes, 0, bytes.length); rowOffsets.put(row.getRowIndex(), bodyOffset); rowIndexes.add(row.getRowIndex()); body.write(bytes); bodyOffset += bytes.length; // pad rows to 8 byte offsets to speed things up. while (bodyOffset % 8 != 0) { bodyOffset++; body.write(ROW_PADDING); } }
public static void write(File file, Iterator<SparseMatrixRow> rows) throws IOException { write(file, rows, new ValueConf()); } public static void write(File file, Iterator<SparseMatrixRow> rows, ValueConf vconf) throws IOException {
public DenseMatrixRow(ValueConf vconf, int rowIndex, int colIds[], float colVals[]) { if (!SparseMatrixUtils.isIncreasing(colIds)) { throw new IllegalArgumentException("Columns must be sorted by id"); } this.vconf = vconf; this.c1 = vconf.c1; this.c2 = vconf.c2; this.colIds = colIds; short packed[] = new short[colVals.length]; for (int i = 0; i < colVals.length; i++) { packed[i] = vconf.pack(colVals[i]); } createBuffer(rowIndex, colIds, packed); }
private void readHeaders() throws IOException { long size = Math.min(channel.size(), DEFAULT_HEADER_SIZE); MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, size); if (buffer.getInt(0) != FILE_HEADER) { throw new IOException("invalid file header: " + buffer.getInt(0)); } this.vconf = new ValueConf(buffer.getFloat(4), buffer.getFloat(8)); this.numRows = buffer.getInt(12); int headerSize = 16 + 12*numRows; if (headerSize > DEFAULT_HEADER_SIZE) { info("maxPageSize not large enough for entire header. Resizing to " + headerSize); buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, headerSize); } debug("preparing buffer for " + numRows + " rows"); buffer.position(16); buffer.limit(buffer.position() + 4 * numRows); rowIds = buffer.slice().asIntBuffer(); if (rowIds.capacity() != numRows) { throw new IllegalStateException(); } buffer.position(16 + 4 * numRows); buffer.limit(buffer.position() + 8 * numRows); rowOffsets = buffer.slice().asLongBuffer(); if (rowOffsets.capacity() != numRows) { throw new IllegalStateException(); } }
private void readHeaders() throws IOException { long size = Math.min(channel.size(), DEFAULT_HEADER_SIZE); MappedByteBuffer buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, size); if (buffer.getInt(0) != FILE_HEADER) { throw new IOException("invalid file header: " + buffer.getInt(0)); } this.vconf = new ValueConf(buffer.getFloat(4), buffer.getFloat(8)); this.numRows = buffer.getInt(12); int headerSize = 16 + 12*numRows; if (headerSize > DEFAULT_HEADER_SIZE) { info("maxPageSize not large enough for entire header. Resizing to " + headerSize); buffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, headerSize); } debug("preparing buffer for " + numRows + " rows"); buffer.position(16); buffer.limit(buffer.position() + 4 * numRows); rowIds = buffer.slice().asIntBuffer(); if (rowIds.capacity() != numRows) { throw new IllegalStateException(); } buffer.position(16 + 4 * numRows); buffer.limit(buffer.position() + 8 * numRows); rowOffsets = buffer.slice().asLongBuffer(); if (rowOffsets.capacity() != numRows) { throw new IllegalStateException(); } }