private void createRemoteClient() throws Exception { remoteClient = SparkClientFactory.createClient(conf, hiveConf); if (HiveConf.getBoolVar(hiveConf, ConfVars.HIVE_PREWARM_ENABLED) && SparkClientUtilities.isYarnMaster(hiveConf.get("spark.master"))) { int minExecutors = getExecutorsToWarm(); if (minExecutors <= 0) { return; } LOG.info("Prewarm Spark executors. The minimum number of executors to warm is " + minExecutors); // Spend at most MAX_PREWARM_TIME to wait for executors to come up. int curExecutors = 0; long ts = System.currentTimeMillis(); do { try { curExecutors = getExecutorCount(MAX_PREWARM_TIME, TimeUnit.MILLISECONDS); } catch (TimeoutException e) { // let's don't fail on future timeout since we have a timeout for pre-warm LOG.warn("Timed out getting executor count.", e); } if (curExecutors >= minExecutors) { LOG.info("Finished prewarming Spark executors. The current number of executors is " + curExecutors); return; } Thread.sleep(500); // sleep half a second } while (System.currentTimeMillis() - ts < MAX_PREWARM_TIME); LOG.info("Timeout (" + MAX_PREWARM_TIME / 1000 + "s) occurred while prewarming executors. " + "The current number of executors is " + curExecutors); } }
(SparkClientUtilities.isYarnMaster(hiveConf.get("spark.master")) || SparkClientUtilities.isLocalMaster(hiveConf.get("spark.master")))) { int minExecutors = getExecutorsToWarm(); if (minExecutors <= 0) { return;