private void loadExisting(File f) { for( File c : f.listFiles() ) { if( c.isDirectory() ) { loadExisting(c); // Recursively keep loading K/V pairs } else { Key k = str2Key(c.getName()); Value ice = new Value(k, (int) c.length()); ice.setdsk(); H2O.putIfAbsent_raw(k, ice); } } }
@Override public void store(Value v) { // Should be used only if ice goes to HDFS assert this == getIce(); assert !v.isPersisted(); byte[] m = v.memOrLoad(); assert (m == null || m.length == v._max); // Assert not saving partial files store(new Path(_iceRoot, getIceName(v)), m); v.setdsk(); // Set as write-complete to disk }
@Override public void store(Value v) { // Only the home node does persistence on NFS if( !v._key.home() ) return; // A perhaps useless cutout: the upper layers should test this first. if( v.isPersisted() ) return; try { File f = getFileForKey(v._key); f.mkdirs(); FileOutputStream s = new FileOutputStream(f); try { byte[] m = v.memOrLoad(); assert (m == null || m.length == v._max); // Assert not saving partial files if( m != null ) new AutoBuffer(s.getChannel(), false, Value.NFS).putA1(m, m.length).close(); v.setdsk(); // Set as write-complete to disk } finally { s.close(); } } catch( IOException e ) { H2O.ignore(e); } }
@Override public void store(Value v) { assert !v.isPersisted(); new File(_dir, getIceDirectory(v._key)).mkdirs(); // Nuke any prior file. FileOutputStream s = null; try { s = new FileOutputStream(getFile(v)); } catch( FileNotFoundException e ) { String info = "Key: " + v._key.toString() + "\nEncoded: " + getFile(v); throw new RuntimeException(Log.err("Encoding a key to a file failed!\n" + info, e)); } try { byte[] m = v.memOrLoad(); // we are not single threaded anymore assert m != null && m.length == v._max : "Trying to save partial file: value key=" + v._key + ", length to save=" + m + ", value max size=" + v._max; // Assert not saving partial files new AutoBuffer(s.getChannel(), false, Value.ICE).putA1(m, m.length).close(); v.setdsk(); // Set as write-complete to disk } finally { Utils.close(s); } }
public static Key loadKey(S3ObjectSummary obj) throws IOException { Key k = encodeKey(obj.getBucketName(), obj.getKey()); long size = obj.getSize(); Value val = new Value(k, (int) size, Value.S3); // Plain Value val.setdsk(); DKV.put(k, val); return k; }
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) { try { if( fs == null ) return; for( FileStatus file : fs.listStatus(p) ) { Path pfs = file.getPath(); if( file.isDir() ) { addFolder(fs, pfs, succeeded, failed); } else { Key k = Key.make(pfs.toString()); long size = file.getLen(); Value val = new Value(k, (int) size, Value.HDFS); // Plain Value val.setdsk(); DKV.put(k, val); Log.info("PersistHdfs: DKV.put(" + k + ")"); JsonObject o = new JsonObject(); o.addProperty(Constants.KEY, k.toString()); o.addProperty(Constants.FILE, pfs.toString()); o.addProperty(Constants.VALUE_SIZE, file.getLen()); succeeded.add(o); } } } catch( Exception e ) { Log.err(e); JsonObject o = new JsonObject(); o.addProperty(Constants.FILE, p.toString()); o.addProperty(Constants.ERROR, e.getMessage()); failed.add(o); } } }
@Override public Value chunkIdx( int cidx ) { final long nchk = nChunks(); assert 0 <= cidx && cidx < nchk; Key dkey = chunkKey(cidx); Value val1 = DKV.get(dkey);// Check for an existing one... will fetch data as needed if( val1 != null ) return val1; // Found an existing one? // Lazily create a DVec for this chunk int len = (int)(cidx < nchk-1 ? CHUNK_SZ : (_len-chunk2StartElem(cidx))); // DVec is just the raw file data with a null-compression scheme Value val2 = new Value(dkey,len,null,TypeMap.C1NCHUNK,_be); val2.setdsk(); // It is already on disk. // If not-home, then block till the Key is everywhere. Most calls here are // from the parser loading a text file, and the parser splits the work such // that most puts here are on home - so this is a simple speed optimization: // do not make a Futures nor block on it on home. Futures fs = dkey.home() ? null : new Futures(); // Atomically insert: fails on a race, but then return the old version Value val3 = DKV.DputIfMatch(dkey,val2,null,fs); if( !dkey.home() && fs != null ) fs.blockForPending(); return val3 == null ? val2 : val3; } }