/** * Compute the fraction of rows that need to be used for training during one iteration * @param numRows number of training rows * @param train_samples_per_iteration number of training rows to be processed per iteration * @param replicate_training_data whether of not the training data is replicated on each node * @return fraction of rows to be used for training during one iteration */ private float computeRowUsageFraction(final long numRows, final long train_samples_per_iteration, final boolean replicate_training_data) { float rowUsageFraction = (float)train_samples_per_iteration / numRows; if (replicate_training_data) rowUsageFraction /= H2O.CLOUD.size(); assert(rowUsageFraction > 0); return rowUsageFraction; } private float rowFraction(Frame train, DeepWaterParameters p, DeepWaterModel m) {
public Maxout(DeepLearningParameters params, short k, int units) { super(units); _k = k; _maxIncoming=new int[params._mini_batch_size][]; for (int i=0;i<_maxIncoming.length;++i) _maxIncoming[i]=new int[units]; if (_k!=2) throw H2O.unimpl("Maxout is currently hardcoded for 2 channels. Trivial to enable k > 2 though."); } @Override protected void fprop(long seed, boolean training, int n) {
public void submitTask() {H2O.submitTask(this);} public void forkTask() {fork();}
/** The main method launched in the H2O environment and * generating documentation. */ public static void main(String[] args) throws Exception { // Boot invoke by default mainClass water.H2O and then call runClass H2O.waitForCloudSize(1); createReSTFilesInCwd(); H2O.exit(0); }
@Override protected int size() { throw H2O.fail(); } @Override protected int numNodes() { throw H2O.fail(); }
if (H2O.CLOUD.size() > 1) { if(H2O.SELF.getSecurityManager().securityEnabled) { throw new H2OIllegalArgumentException("Cannot run XGBoost on an SSL enabled cluster larger than 1 node. XGBoost does not support SSL encryption."); if (H2O.getCloudSize() > 1) error("_backend", "GPU backend is not supported in distributed mode."); case bernoulli: if( _nclass != 2 /*&& !couldBeBool(_response)*/) error("_distribution", technote(2, "Binomial requires the response to be a 2-class categorical")); break; case modified_huber: if( _nclass != 2 /*&& !couldBeBool(_response)*/) error("_distribution", technote(2, "Modified Huber requires the response to be a 2-class categorical.")); break; case multinomial: if (!isClassifier()) error("_distribution", technote(2, "Multinomial requires an categorical response.")); break; case huber: if (isClassifier()) error("_distribution", technote(2, "Huber requires the response to be numeric.")); break; case poisson: if (isClassifier()) error("_distribution", technote(2, "Poisson requires the response to be numeric.")); break; case gamma: if (isClassifier()) error("_distribution", technote(2, "Gamma requires the response to be numeric.")); break; case tweedie: if (isClassifier()) error("_distribution", technote(2, "Tweedie requires the response to be numeric."));
if (_l2 < 0) dl.error("_l2", "L2 penalty must be >= 0."); if (H2O.CLOUD.size() == 1 && _replicate_training_data) dl.hide("_replicate_training_data", "replicate_training_data is only valid with cloud size greater than 1."); if (_single_node_mode && (H2O.CLOUD.size() == 1 || !_replicate_training_data)) dl.hide("_single_node_mode", "single_node_mode is only used with multi-node operation with replicated training data."); if (H2O.ARGS.client && _single_node_mode) dl.error("_loss", "Cannot use CrossEntropy loss for auto-encoder."); if (!classification && _loss == CrossEntropy) dl.error("_loss", technote(2, "For CrossEntropy loss, the response must be categorical.")); case gamma: case poisson: dl.error("_distribution", technote(2, _distribution + " distribution is not allowed for classification.")); break; case AUTO: case bernoulli: case modified_huber: dl.error("_distribution", technote(2, _distribution + " distribution is not allowed for regression.")); break; case tweedie: dl.error("_class_sampling_factors", "class_sampling_factors requires balance_classes to be enabled."); if (_replicate_training_data && null != train() && train().byteSize() > 0.9*H2O.CLOUD.free_mem()/H2O.CLOUD.size() && H2O.CLOUD.size() > 1) { dl.error("_replicate_training_data", "Compressed training dataset takes more than 90% of avg. free available memory per node (" + 0.9*H2O.CLOUD.free_mem()/H2O.CLOUD.size() + "), cannot run with replicate_training_data.");
public static void waitForCloudSize(int x) { waitForCloudSize(x, 10000); }
static public int onIce(String className) { Integer I = MAP.get(className); if( I != null ) return I; // Need to install a new cloud-wide type ID for className assert H2O.CLOUD.size() > 0 : "No cloud when getting type id for "+className; int id = -1; if( H2O.CLOUD.leader() != H2O.SELF ) // Not leader? id = FetchId.fetchId(className); return install(className,id); }
public DeepwaterMojoWriter(DeepWaterModel model) { super(model); _parms = model.get_params(); _model_info = model.model_info(); _output = model._output; if (_model_info._unstable) { throw new UnsupportedOperationException(technote(4, "Refusing to create a MOJO for an unstable model.")); } }
@BeforeClass public static void setupCloud() { H2O.main(new String[] { }); }
if (H2O.getCloudSize() != 1) throw new IllegalArgumentException("Deep Water currently only supports execution of 1 node."); _output._origNames = params._train.get().names(); throw new IllegalArgumentException(technote(5, "Model is too large to fit into the DKV (larger than " + PrettyPrint.bytes(Value.MAX) + ")."));
@Before public void initCloud() { // Setup cloud name String[] args = new String[] { "-name", "h2o_test_cloud"}; // Build a cloud of 1 H2O.main(args); H2O.waitForCloudSize(1, 10*1000 /* ms */); }
@BeforeClass public static void setupCloud() { H2O.main(new String[] {}); _initial_keycnt = H2O.store_size(); assert Job.all().length == 0; // No outstanding jobs _testClassTimer = new Timer(); }
String[] workerArgs = new String[] { "-flatfile", FLATFILE, "-port", "" + PORT }; List<FlatFileEntry> flatfile = H2O.parseFlatFile(new File(FLATFILE)); HashMap<String, Host> hosts = new HashMap<String, Host>(); ArrayList<Node> workers = new ArrayList<Node>(); w.start(); H2O.main(Utils.append(workerArgs, args)); stall_till_cloudsize(1 + workers.size(), 10000); // stall for cloud 10seconds Log.unwrap(System.out, "");
if (H2O.getCloudSize() > 1) { Log.info("GPU backend not supported in distributed mode. Using CPU backend."); } else if (! p.gpuIncompatibleParams().isEmpty()) { Log.info("GPU backend not supported for the choice of parameters (" + p.gpuIncompatibleParams() + "). Using CPU backend."); } else if (XGBoost.hasGPU(H2O.CLOUD.members()[0], p._gpu_id)) { Log.info("Using GPU backend (gpu_id: " + p._gpu_id + ")."); params.put("gpu_id", p._gpu_id);
final void buildModel() { if ((XGBoostModel.XGBoostParameters.Backend.auto.equals(_parms._backend) || XGBoostModel.XGBoostParameters.Backend.gpu.equals(_parms._backend)) && hasGPU(_parms._gpu_id) && H2O.getCloudSize() == 1 && _parms.gpuIncompatibleParams().isEmpty()) { synchronized (XGBoostGPULock.lock(_parms._gpu_id)) { buildModelImpl(); } } else { buildModelImpl(); } }
@Override protected AutoBuffer compress(AutoBuffer ab, AutoBuffer abAux) { throw H2O.fail(); } @Override protected int size() { throw H2O.fail(); }
public static void stall_till_cloudsize(int x, long ms) { H2O.waitForCloudSize(x, ms); UKV.put(Job.LIST, new Job.List()); // Jobs.LIST must be part of initial keys } }