public void setData(File f, boolean loadNow) throws IOException { Preconditions.checkArgument(f.length() == rows * columns * 8L, "File " + f + " is wrong length"); for (int i = 0; i < (rows + rowsPerBlock - 1) / rowsPerBlock; i++) { long start = i * rowsPerBlock * columns * 8L; long size = rowsPerBlock * columns * 8L; MappedByteBuffer buf = new FileInputStream(f).getChannel().map(FileChannel.MapMode.READ_ONLY, start, Math.min(f.length() - start, size)); if (loadNow) { buf.load(); } addData(buf.asDoubleBuffer()); } }
public DoubleBigMappedVector(MappedByteBuffer bytes, int capacity) { this(bytes.asDoubleBuffer(), capacity); }
private void create(File file) throws IOException { _raf = new RandomAccessFile(file, "rw"); // 3 ints + (_1stDim * _2ndDim * 3rdDim) doubles _raf.setLength((_firstDim * _secondDim * _thirdDim * 8) + 12); _filechannel = _raf.getChannel(); _bytebuffer = _filechannel.map(FileChannel.MapMode.READ_WRITE, 0, (int) _filechannel.size()); _bytebuffer.position(12); _mmap = _bytebuffer.asDoubleBuffer(); }
public static DoubleBuffer memmapFlatLibrariesBuf( File flatDir ) throws IOException { return memmapReadOnly( new File( flatDir, flatLibrariesFilename ) ).asDoubleBuffer( ); }
public static DoubleBuffer memmapFlatVerticesBuf( File flatDir ) throws IOException { return memmapReadOnly( new File( flatDir, flatVerticesFilename ) ).asDoubleBuffer( ); }
@Override public void init(int capacity) { try { if (data == null) { this.capacity = capacity; this.file = File.createTempFile("matrix", ".mapped"); this.file.deleteOnExit(); RandomAccessFile raf = new RandomAccessFile(file, "rw"); data = raf.getChannel() .map(FileChannel.MapMode.READ_WRITE, 0, capacity * DOUBLE_SIZE) .asDoubleBuffer(); } } catch (IOException eee) { throw new MatrixException("Can't init vector", eee); } }
public void setData(File f, boolean loadNow) throws IOException { Preconditions.checkArgument(f.length() == rows * columns * 8L, "File " + f + " is wrong length"); for (int i = 0; i < (rows + rowsPerBlock - 1) / rowsPerBlock; i++) { long start = i * rowsPerBlock * columns * 8L; long size = rowsPerBlock * columns * 8L; MappedByteBuffer buf = new FileInputStream(f).getChannel().map(FileChannel.MapMode.READ_ONLY, start, Math.min(f.length() - start, size)); if (loadNow) { buf.load(); } addData(buf.asDoubleBuffer()); } }
/** * Utilise une partie du fichier pour stocker les informations * @param raf le fichier a utiliser * @param offset l'endroit ou l'on doit faire le stockage * @param capacity le nombre de double a y stocker * @throws IOException */ public DoubleBigMappedVector(RandomAccessFile raf, long offset, int capacity) throws IOException { this.capacity = capacity; try { this.data = raf.getChannel().map( FileChannel.MapMode.READ_WRITE, offset, capacity*DOUBLE_SIZE).asDoubleBuffer(); } catch (Exception eee) { // can't use mapped version, read array and create not mapped Buffer log.error("Can't use mapped file, only read available"); this.readonly = true; byte[] tmp = new byte[capacity*DOUBLE_SIZE]; long currentOffset = raf.getFilePointer(); raf.seek(offset); raf.readFully(tmp); raf.seek(currentOffset); ByteBuffer buf = ByteBuffer.wrap(tmp); this.data = buf.asDoubleBuffer(); } }
@Override public final Array<Double> expand(int newLength) { try { if (newLength > length) { this.buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, BYTE_COUNT * newLength).asDoubleBuffer(); this.fill(defaultValue, length, newLength); this.length = newLength; } return this; } catch (Exception ex) { throw new ArrayException("Failed to expand size of memory mapped array at " + file.getAbsolutePath(), ex); } }
/** * * @param size the size of the buffer in bytes */ private static Duple<DoubleBuffer,File> createTempBuffer(int size) { try { File f = File.createTempFile("OnDiskMatrix",".matrix"); // Make sure the temp file goes away since it can get fairly large // for big matrices f.deleteOnExit(); RandomAccessFile raf = new RandomAccessFile(f, "rw"); FileChannel fc = raf.getChannel(); DoubleBuffer contextBuffer = fc.map(MapMode.READ_WRITE, 0, size).asDoubleBuffer(); fc.close(); return new Duple<DoubleBuffer,File>(contextBuffer, f); } catch (IOException ioe) { throw new IOError(ioe); } }
/** * * @param size the size of the buffer in bytes */ private static Duple<DoubleBuffer,File> createTempBuffer(int size) { try { File f = File.createTempFile("OnDiskMatrix",".matrix"); // Make sure the temp file goes away since it can get fairly large // for big matrices f.deleteOnExit(); RandomAccessFile raf = new RandomAccessFile(f, "rw"); FileChannel fc = raf.getChannel(); DoubleBuffer contextBuffer = fc.map(MapMode.READ_WRITE, 0, size).asDoubleBuffer(); fc.close(); return new Duple<DoubleBuffer,File>(contextBuffer, f); } catch (IOException ioe) { throw new IOError(ioe); } }
/** * Constructor * @param length the length of the array * @param defaultValue the default value for array * @param file the memory mapped file reference */ MappedArrayOfDoubles(int length, Double defaultValue, File file) { super(Double.class, ArrayStyle.MAPPED, false); try { this.file = file; this.length = length; this.defaultValue = defaultValue == null ? Double.NaN : defaultValue; this.channel = new RandomAccessFile(file, "rw").getChannel(); this.buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, BYTE_COUNT * length).asDoubleBuffer(); this.fill(defaultValue); } catch (Exception ex) { throw new ArrayException("Failed to initialise memory mapped array on file: " + file.getAbsolutePath(), ex); } }
@SuppressWarnings("unchecked") /** Custom serialization */ private void readObject(ObjectInputStream is) throws IOException, ClassNotFoundException { this.file = MappedArrayConstructor.randomFile(true); this.length = is.readInt(); this.defaultValue = is.readDouble(); this.channel = new RandomAccessFile(file, "rw").getChannel(); this.buffer = channel.map(FileChannel.MapMode.READ_WRITE, 0, BYTE_COUNT * length).asDoubleBuffer(); for (int i=0; i<length; ++i) { final double value = is.readDouble(); this.setDouble(i, value); } }
public void open(String path, String filename, boolean overwrite) throws IOException { if (overwrite) throw new IOException("Can't overwrite: readonly access"); Os.createDirectory(new File(path)); String fullpath = path + Os.pathSeparator() + filename; _name = filename; File file = new File(fullpath); _file = new RandomAccessFile(file, "r"); _filechannel = _file.getChannel(); _bytebuffer = _filechannel.map(FileChannel.MapMode.READ_ONLY, 0, (int) _filechannel.size()); _firstDim = _bytebuffer.getInt(); _secondDim = _bytebuffer.getInt(); _thirdDim = _bytebuffer.getInt(); _mmap = _bytebuffer.asDoubleBuffer(); }
public void open(String path, String filename, boolean overwrite) throws IOException { Os.createDirectory(new File(path)); String fullpath = path + Os.pathSeparator() + filename; _name = filename; File file = new File(fullpath); if (file.exists()) { if (overwrite) { file.delete(); create(file); } else { _raf = new RandomAccessFile(file, "rw"); _filechannel = _raf.getChannel(); _bytebuffer = _filechannel.map(FileChannel.MapMode.READ_WRITE, 0, (int) _filechannel.size()); _firstDim = _bytebuffer.getInt(); _secondDim = _bytebuffer.getInt(); _thirdDim = _bytebuffer.getInt(); _mmap = _bytebuffer.asDoubleBuffer(); } } else create(file); }
tmpMappedByteBuffer.order(ByteOrder.nativeOrder()); tmpDoubleBuffer = tmpMappedByteBuffer.asDoubleBuffer();
tmpMappedByteBuffer.order(ByteOrder.nativeOrder()); tmpDoubleBuffer = tmpMappedByteBuffer.asDoubleBuffer();
int totalLibrariesByteCount = totalLibraryCount * doublesPerFlatLibrary * SIZEOF_DOUBLE; MappedByteBuffer librariesMapped = createAndMemmapReadWrite( librariesFile, totalLibrariesByteCount ); DoubleBuffer librariesBuf = librariesMapped.asDoubleBuffer( ); DoubleBuffer verticesBuf = verticesMapped.asDoubleBuffer( );