Col(String s, int rows, boolean isClass) { _name = s; _isClass = isClass; _rawB = MemoryManager.malloc1(rows); _isFloat = false; _isByte = true; _colBinLimit = 0; }
public AppendableVec( Key key, long [] espc, int chunkOff) { super(key, (long[])null); _espc = espc; _chunkTypes = MemoryManager.malloc1(espc.length); _chunkOff = chunkOff; }
public static byte [] malloc1 (int size) { return malloc1(size,false); } public static byte [] malloc1 (int size, boolean force) { return (byte [])malloc(size,size*1, 1,null,0,force); }
public UDPPing(int sz){ assert sz <= AutoBuffer.MTU:"msg size does not fit into UDP"; _payload = MemoryManager.malloc1(sz); Random rnd = new Random(); for(int i = 0; i < _payload.length; ++i) _payload[i] = (byte)rnd.nextInt(); _t1 = System.currentTimeMillis(); } @Override
/** Write to a known sized byte[]. Instead of calling close(), call * {@link #bufClose()} to retrieve the final byte[]. */ public AutoBuffer( int len ) { _bb = ByteBuffer.wrap(MemoryManager.malloc1(len)).order(ByteOrder.nativeOrder()); _chan = null; _h2o = null; _read = false; _firstPage = true; _persist = 0; // No persistance }
public C8DChunk (double [] vals){ _start = -1; _mem = MemoryManager.malloc1(vals.length << 3); _len = vals.length; for(int i = 0, off = 0; i < vals.length; ++i, off += 8) UDP.set8d(_mem,off,vals[i]); } C8DChunk( byte[] bs ) { _mem=bs; _start = -1; _len = _mem.length>>3; }
public byte[] getA1( int len ) { byte[] buf = MemoryManager.malloc1(len); int sofar = 0; while( sofar < len ) { int more = Math.min(_bb.remaining(), len - sofar); _bb.get(buf, sofar, more); sofar += more; if( sofar < len ) getSp(Math.min(_bb.capacity(), len-sofar)); } return buf; }
public FVecDataOut(VectorGroup vg, int chunkOff, int cidx, Enum [] enums, AppendableVec [] appendables){ assert cidx - chunkOff >= 0:"incompatible cidx/chunkOff " + chunkOff + ", " + cidx; _vecs = appendables; _chunkOff = chunkOff; _enums = enums; _cidx = cidx; _vg = vg; _ctypes = MemoryManager.malloc1(appendables.length); _nvs = new NewChunk[appendables.length]; for(int i = 0; i < appendables.length; ++i) _nvs[i] = (NewChunk)_vecs[i].chunkForChunkIdx(_cidx); }
@SuppressWarnings("deprecation") public AutoBuffer putStr( String s ) { if( s==null ) return putInt(-1); // Use the explicit getBytes instead of the default no-arg one, to avoid // the overhead of going in an out of a charset decoder. byte[] buf = MemoryManager.malloc1(s.length()); s.getBytes(0,buf.length,buf,0); return putA1(buf); }
/** Make a new Key that fits the requirements for a Vec key, based on the * passed-in key. Used to make Vecs that back over e.g. disk files. */ static Key newKey(Key k) { byte [] kb = k._kb; byte [] bits = MemoryManager.malloc1(kb.length+KEY_PREFIX_LEN); bits[0] = Key.VEC; bits[1] = -1; // Not homed UDP.set4(bits,2,0); // new group, so we're the first vector UDP.set4(bits,6,-1); // 0xFFFFFFFF in the chunk# area System.arraycopy(kb, 0, bits, 4+4+1+1, kb.length); return Key.make(bits); }
private Chunk chunkUUID() { final byte [] bs = MemoryManager.malloc1(_len*16,true); int j = 0; for( int i = 0; i < _len; ++i ) { long lo = 0, hi=0; if(_id == null || _id.length == 0 || (j < _id.length && _id[j] == i ) ) { lo = _ls[j]; hi = Double.doubleToRawLongBits(_ds[j++]); if( _xs != null && _xs[j] == Integer.MAX_VALUE){// NA? lo = Long.MIN_VALUE; hi = 0; // Canonical NA value } } UDP.set8(bs, 16*i , lo); UDP.set8(bs, 16*i+8, hi); } assert j == _sparseLen:"j = " + j + ", _sparseLen = " + _sparseLen; return new C16Chunk(bs); }
@Override protected void writeModelData() throws IOException { writekv("vec_size", model._parms._vec_size); writekv("vocab_size", model._output._words.length); // Vocabulary startWritingTextFile("vocabulary"); for (BufferedString word : model._output._words) { writeln(word.toString(), true); } finishWritingTextFile(); // Vectors ByteBuffer bb = ByteBuffer.wrap(MemoryManager.malloc1(model._output._vecs.length * 4)); for (float v : model._output._vecs) bb.putFloat(v); writeblob("vectors", bb.array()); }
@Override public byte[] load(final Value v) { final byte[] b = MemoryManager.malloc1(v._max); long skip = 0; Key k = v._key; if(k._kb[0] == Key.DVEC) skip = FileVec.chunkOffset(k); // The offset final Path p = _iceRoot == null?new Path(getPathForKey(k)):new Path(_iceRoot, getIceName(v)); final long skip_ = skip; run(new Callable() { @Override public Object call() throws Exception { FileSystem fs = FileSystem.get(p.toUri(), CONF); FSDataInputStream s = null; try { s = fs.open(p); // NOTE: // The following line degrades performance of HDFS load from S3 API: s.readFully(skip,b,0,b.length); // Google API's simple seek has better performance // Load of 300MB file via Google API ~ 14sec, via s.readFully ~ 5min (under the same condition) ByteStreams.skipFully(s, skip_); ByteStreams.readFully(s, b); assert v.isPersisted(); } finally { Utils.close(s); } return null; } }, true, v._max); return b; }
@Override public Model load(Path f) throws IOException { FSDataInputStream is = _hfs.open(f); byte buf[] = MemoryManager.malloc1((int) _hfs.getContentSummary(f).getLength()); try { is.readFully(buf); } finally { is.close(); } AutoBuffer ab=ab4read(buf); Model m = loadHeader(ab); m.getModelSerializer().load(m, ab); if (m._key!=null) { DKV.put(m._key, m); } return m; }
public AppendableVec[] vecs(){ if(_dout.length == 1) return _dout[0]._vecs; int nCols = 0; for(FVecDataOut dout:_dout) nCols = Math.max(dout._vecs.length,nCols); AppendableVec [] res = new AppendableVec[nCols]; int nchunks = 0; for(FVecDataOut dout:_dout) nchunks += dout.nChunks(); long [] espc = MemoryManager.malloc8(nchunks); for(int i = 0; i < res.length; ++i) { res[i] = new AppendableVec(_vg.vecKey(_vecIdStart + i), espc, 0); res[i]._chunkTypes = MemoryManager.malloc1(nchunks); } for(int i = 0; i < _dout.length; ++i) for(int j = 0; j < _dout[i]._vecs.length; ++j) res[j].setSubRange(_dout[i]._vecs[j]); return res; } public String _parserr; // NULL if parse is OK, else an error string
@Override public byte[] load(Value v) { long start_io_ms = System.currentTimeMillis(); byte[] b = MemoryManager.malloc1(v._max); Key k = v._key; long skip = 0;
@Override public byte[] load(Value v) { Key k = v._key; // key for value if (k._kb[0] != Key.DVEC) throw H2O.unimpl(); // Load only from values stored in vector long skip = FileVec.chunkOffset(k); // Compute skip for this value long start_io_ms = System.currentTimeMillis(); final byte[] b = MemoryManager.malloc1(v._max); String[] keyComp = decodeKey(k); String clientUri = keyComp[0]; String fpath = keyComp[1]; TachyonFS tfs = null; InputStream is = null; try { tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientUri)); long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); ByteStreams.skipFully(is, skip); ByteStreams.readFully(is, b); TimeLine.record_IOclose(start_ns, start_io_ms, 1/* read */, v._max, Value.TACHYON); return b; } catch (IOException e) { throw new RuntimeException(Log.err("File load failed: ", e)); } finally { if (is!=null) Utils.close(is); } }
private Chunk chunkD() { if (H2O.SINGLE_PRECISION) { final byte[] bs = MemoryManager.malloc1(_len * 4, true); int j = 0; for (int i = 0; i < _len; ++i) { float f = 0; if (_id == null || _id.length == 0 || (j < _id.length && _id[j] == i)) { f = _ds != null ? (float)_ds[j] : (isNA2(j) || isEnum(j)) ? Float.NaN : (float)(_ls[j] * PrettyPrint.pow10(_xs[j])); ++j; } UDP.set4f(bs, 4 * i, f); } assert j == _sparseLen : "j = " + j + ", _sparseLen = " + _sparseLen; return new C4FChunk(bs); } else { final byte[] bs = MemoryManager.malloc1(_len * 8, true); int j = 0; for (int i = 0; i < _len; ++i) { double d = 0; if (_id == null || _id.length == 0 || (j < _id.length && _id[j] == i)) { d = _ds != null ? _ds[j] : (isNA2(j) || isEnum(j)) ? Double.NaN : _ls[j] * PrettyPrint.pow10(_xs[j]); ++j; } UDP.set8d(bs, 8 * i, d); } assert j == _sparseLen : "j = " + j + ", _sparseLen = " + _sparseLen; return new C8DChunk(bs); } }
private byte[] bufD(final int valsz){ int log = 0; while((1 << log) < valsz)++log; assert (1 << log) == valsz; final int ridsz = _len >= 65535?4:2; final int elmsz = ridsz + valsz; int off = CXDChunk.OFF; byte [] buf = MemoryManager.malloc1(off + _sparseLen*elmsz,true); for( int i=0; i<_sparseLen; i++, off += elmsz ) { if(ridsz == 2) UDP.set2(buf,off,(short)_id[i]); else UDP.set4(buf,off,_id[i]); final double dval = _ds == null?isNA2(i)?Double.NaN:_ls[i]*PrettyPrint.pow10(_xs[i]):_ds[i]; switch(valsz){ case 4: UDP.set4f(buf, off + ridsz, (float) dval); break; case 8: UDP.set8d(buf, off + ridsz, dval); break; default: throw H2O.unimpl(); } } assert off==buf.length; return buf; } // Compute a compressed integer buffer
int off = (equal == 3) ? _ts.get2() : 0; int sz = (equal == 3) ? _ts.get2() : 4; byte[] buf = MemoryManager.malloc1(sz); _ts.read(buf, 0, sz); gcmp = new IcedBitSet(buf, sz << 3, off);