@Override protected void setupLocal(){ _dinfo = DKV.get(_dinfoKey).get(); }
@Override public Value atomic(Value val) { T old = val == null ? null : (T)(val.get().clone()); T nnn = atomic(old); // Atomic operation changes the data, so it can not be performed over values persisted on read-only data source // as we would not be able to write those changes back. assert val == null || val.onICE() || !val.isPersisted(); return nnn == null ? null : new Value(_key,nnn,val==null?Value.ICE:(byte)(val._persist&Value.BACKEND_MASK)); } @Override public void onSuccess( Value old ) { onSuccess(old==null?null:(T)old.get()); }
@Override public boolean filter(KeySnapshot.KeyInfo k) { return Value.isSubclassOf(k._type, Leaderboard.class); } }).keys();
/** Set persistence to HDFS from ICE */ public void setHdfs() { assert onICE(); byte[] mem = memOrLoad(); // Get into stable memory _persist = Value.HDFS|Value.NOTdsk; Persist.I[Value.HDFS].store(this); removeIce(); // Remove from ICE disk assert onHDFS(); // Flip to HDFS _mem = mem; // Close a race with the H2O cleaner zapping _mem while removing from ice }
public byte[] getFirstBytes() { Value v = this; if(isByteVec()){ ByteVec vec = get(); return vec.chunkForChunkIdx(0).getBytes(); } else if(isFrame()){ Frame fr = get(); return ((ByteVec)fr.vecs()[0]).chunkForChunkIdx(0).getBytes(); } // Return empty array if key has been deleted return v != null ? v.memOrLoad() : new byte[0]; }
public KeyInfo(Key k, Value v){ assert k!=null : "Key should be not null!"; assert v!=null : "Value should be not null!"; _key = k; _type = v.type(); _rawData = v.isRawData(); if(v.isFrame()){ Frame f = v.get(); // NOTE: can't get byteSize here as it may invoke RollupStats! :( // _sz = f.byteSize(); _sz = v._max; // do at least nrows/ncols instead _ncols = f.numCols(); _nrows = f.numRows(); } else { _sz = v._max; _ncols = 0; _nrows = 0; } _backEnd = v.backend(); } @Override public int compareTo(KeyInfo ki){ return _key.compareTo(ki._key);}
@Override public Value makeValue( Object key, byte[] bits ) { return new Value((Key)key,bits); } @Override public void put( Object key, Object val ) { UKV.put((Key)key,(Value)val); }
@Override public void store(Value v) { // Should be used only if ice goes to HDFS assert this == getIce(); assert !v.isPersisted(); byte[] m = v.memOrLoad(); assert (m == null || m.length == v._max); // Assert not saving partial files store(new Path(_iceRoot, getIceName(v)), m); v.setdsk(); // Set as write-complete to disk }
public static void read_lock( Key k, Key job_key ) { Value val = DKV.get(k); if( val.isLockable() ) ((Lockable)val.get()).read_lock(job_key); // Lockable being locked } public void read_lock( Key job_key ) {
if( !(ov instanceof Value) ) continue; // Ignore tombstones and Primes and null's Value val = (Value)ov; byte[] m = val.rawMem(); Object p = val.rawPOJO(); if( m == null && p == null ) continue; // Nothing to throw out if( val.isLockable() ) continue; // we do not want to throw out Lockables. boolean isChunk = p instanceof Chunk; if( val.isPersisted() && m != null && p != null && !isChunk ) { val.freeMem(); // Toss serialized form, since can rebuild from POJO freed += val._max; if( !val.isPersisted() && !diskFull && (force || (lazyPersist() && lazy_clean(key)))) { try { val.storePersist(); // Write to disk if( m == null ) m = val.rawMem(); if( m != null ) cleaned += m.length; } catch(IOException e) { if( force && val.isPersisted() ) { val.freeMem (); if( m != null ) freed += val._max; m = null; val.freePOJO(); if( p != null ) freed += val._max; p = null; if( isChunk ) freed -= val._max; // Double-counted freed mem for Chunks since val._pojo._mem & val._mem are the same. val.freeMem(); freed += val._max;
/** Creates a Stream for reading bytes */ public InputStream openStream(ProgressMonitor p) throws IOException { if(onNFS() ) return PersistNFS .openStream(_key ); if(onHDFS()) return PersistHdfs.openStream(_key,p); if(onS3() ) return PersistS3 .openStream(_key,p); if(onTachyon()) return PersistTachyon.openStream(_key,p); if( isFrame() ) throw new IllegalArgumentException("Tried to pass a Frame to openStream (maybe tried to parse a (already-parsed) Frame?)"); assert _type==TypeMap.PRIM_B : "Expected byte[] type but got "+TypeMap.className(_type); return new ByteArrayInputStream(memOrLoad()); }
@Override public byte[] getBytes( Object val ) { return ((Value)val).memOrLoad(); }
public final void freeMem() { assert isPersisted() || _pojo != null || _key._kb[0]==Key.DVEC; _mem = null; } // Free POJO (but always be able to rebuild the POJO)
/** Remove dead Values from disk */ void removeIce() { // do not yank memory, as we could have a racing get hold on to this // free_mem(); if( !isPersisted() || !onICE() ) return; // Never hit disk? clrdsk(); // Not persisted now Persist.I[backend()].delete(this); } /** Load some or all of completely persisted Values */