protected void serveHdfs() throws IOException{ if (isBareS3NBucketWithoutTrailingSlash(path)) { path += "/"; } Log.info("ImportHDFS processing (" + path + ")"); ArrayList<String> succ = new ArrayList<String>(); ArrayList<String> fail = new ArrayList<String>(); PersistHdfs.addFolder2(new Path(path), succ, fail); keys = succ.toArray(new String[succ.size()]); files = keys; fails = fail.toArray(new String[fail.size()]); this.prefix = getCommonPrefix(keys); DKV.write_barrier(); }
private void cancel(final String msg, JobState resultingState ) { if(resultingState == JobState.CANCELLED) { Log.info("Job " + self() + "(" + description + ") was cancelled."); } else { Log.err("Job " + self() + "(" + description + ") failed."); Log.err(msg); } exception = msg; state = resultingState; // replace finished job by a job handle replaceByJobHandle(); DKV.write_barrier(); final Job job = this; H2O.submitTask(new H2OCountedCompleter() { @Override public void compute2() { job.onCancelled(); } }); }
DKV.write_barrier(); int leaked_keys = H2O.store_size() - _initial_keycnt; int nvecs = 0, nchunks = 0, nframes = 0, nmodels = 0, nothers = 0;
@Override protected Response serve() { String pstr = _path.value(); if (isBareS3NBucketWithoutTrailingSlash(_path.value())) { pstr = pstr + "/"; } Log.info("ImportHDFS processing (" + pstr + ")"); JsonArray succ = new JsonArray(); JsonArray fail = new JsonArray(); try { PersistHdfs.addFolder(new Path(pstr), succ, fail); } catch( IOException e ) { return Response.error(e); } DKV.write_barrier(); JsonObject json = new JsonObject(); json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size())); json.add(SUCCEEDED, succ); json.add(NUM_FAILED, new JsonPrimitive(fail.size())); json.add(FAILED, fail); Response r = Response.done(json); r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder()); // Add quick link if (succ.size() > 1) r.addHeader("<div class='alert'>" // + Parse.link("*"+pstr+"*", "Parse all into hex format") + " </div>"); return r; } }
@Test public void testRemoteBitSet() throws Exception { Scope.enter(); // Issue a slew of remote key puts Key[] keys = new Key[32]; for( int i = 0; i < keys.length; ++i ) { Key k = keys[i] = Key.make("key"+i); byte[] bits = new byte[4]; bits[0] = (byte)i; // Each value holds a shift-count Value val = new Value(k,bits); DKV.put(k,val); } DKV.write_barrier(); RemoteBitSet r = new RemoteBitSet(); r.invoke(keys); assertEquals((int)((1L<<keys.length)-1), r._x); //for( Key k : keys ) DKV.remove(k); Scope.exit(); }
@Override protected Response serve() { String bucket = _bucket.value(); Log.info("ImportS3 processing (" + bucket + ")"); JsonObject json = new JsonObject(); JsonArray succ = new JsonArray(); JsonArray fail = new JsonArray(); AmazonS3 s3 = PersistS3.getClient(); ObjectListing currentList = s3.listObjects(bucket); processListing(currentList, succ, fail); while(currentList.isTruncated()){ currentList = s3.listNextBatchOfObjects(currentList); processListing(currentList, succ, fail); } json.add(NUM_SUCCEEDED, new JsonPrimitive(succ.size())); json.add(SUCCEEDED, succ); json.add(NUM_FAILED, new JsonPrimitive(fail.size())); json.add(FAILED, fail); DKV.write_barrier(); Response r = Response.done(json); r.setBuilder(SUCCEEDED + "." + KEY, new KeyCellBuilder()); return r; } }
@Test public void testRemoteAtomic() { // Make an execution key homed to the remote node H2O cloud = H2O.CLOUD; H2ONode target = cloud._memary[0]; if( target == H2O.SELF ) target = cloud._memary[1]; Key key = Key.make("test6_remote",(byte)1,Key.DFJ_INTERNAL_USER,target); // It's a plain empty byte array - but too big for atomic update on purpose Value v1 = new Value(key,16); // Remote-put operation DKV.put(key,v1); DKV.write_barrier(); // Atomically run this function on a clone of the bits from the existing // Key and install the result as the new Value. This function may run // multiple times if there are collisions. Atomic q = new Atomic2(); q.invoke(key); // Run remotely; block till done Value val3 = DKV.get(key); assertNotSame(v1,val3); AutoBuffer ab = new AutoBuffer(val3.memOrLoad()); assertEquals(2,ab.get8(0)); assertEquals(2,ab.get8(8)); DKV.remove(key); // Cleanup after test }