@Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM");
Configuration flinkConf = new Configuration(); YarnConfiguration yarnConf = mock(YarnConfiguration.class); doAnswer(getDefault).when(yarnConf).get(anyString(), anyString()); doAnswer(getDefault).when(yarnConf).getInt(anyString(), anyInt()); doAnswer(new Answer() {
XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + applicationAttemptID.getApplicationId().toString() + "/" + applicationAttemptID.getApplicationId().toString()); Path jobLogPath = new Path(xlearningConf.get("fs.defaultFS"), logdir); LOG.info("jobLogPath:" + jobLogPath.toString()); LOG.info("Start write the log to " + jobLogPath.toString()); XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + conf.get("hadoop.job.ugi").split(",")[0] + "/" + envs.get(XLearningConstants.Environment.XLEARNING_APP_NAME.toString())); Path hostLocalPath = new Path(xlearningConf.get("fs.defaultFS"), hostLocaldir); try { FSDataOutputStream hostLocalOut = fs.create(hostLocalPath); Path boardLogPath; if (conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_LOG_DIR).indexOf("hdfs://") == -1) { if (conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR).equals(xlearningConf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR))) { boardLogPath = new Path(xlearningConf.get("fs.defaultFS"), conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR) + "/" + applicationAttemptID.getApplicationId().toString()); } else {
private static HAServiceState getHAState(YarnConfiguration yarnConf) throws Exception { HAServiceTarget haServiceTarget; int rpcTimeoutForChecks = yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, yarnConf.get(YarnConfiguration.RM_PRINCIPAL, "")); haServiceTarget = new RMHAServiceTarget(yarnConf); HAServiceProtocol proto = haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); HAServiceState haState = proto.getServiceStatus().getState(); return haState; }
/** * Get the socket address for <code>name</code> property as a * <code>InetSocketAddress</code>. On an HA cluster, * this fetches the address corresponding to the RM identified by * {@link #RM_HA_ID}. * @param name property name. * @param defaultAddress the default value * @param defaultPort the default port * @return InetSocketAddress */ @Override public InetSocketAddress getSocketAddr( String name, String defaultAddress, int defaultPort) { String address; if (HAUtil.isHAEnabled(this) && getServiceAddressConfKeys(this).contains(name)) { address = HAUtil.getConfValueForRMInstance(name, defaultAddress, this); } else { address = get(name, defaultAddress); } return NetUtils.createSocketAddr(address, defaultPort, name); }
public Executor(String instancName, String shellCommand, STARTType startType, String runningContainer, String localDir, String deployPath, String hadoopHome, String javaHome, String pythonHome, String dstPath, String portList, String shellArgs, String ExecShellStringPath, String applicationId, String supervisorLogviewPort, String nimbusThriftPort) { executorMeta = new ExecutorMeta(instancName, shellCommand, startType, runningContainer, localDir, deployPath, hadoopHome, javaHome, pythonHome, dstPath, portList, shellArgs, ExecShellStringPath, applicationId, supervisorLogviewPort, nimbusThriftPort); conf = new YarnConfiguration(); Path yarnSite = new Path(hadoopHome + JOYConstants.YARN_SITE_PATH); conf.addResource(yarnSite); //get first log dir logDir = conf.get(JOYConstants.YARN_NM_LOG, JOYConstants.YARN_NM_LOG_DIR).split(JOYConstants.COMMA)[0] + JOYConstants.BACKLASH + applicationId + JOYConstants.BACKLASH + runningContainer; //Setup RegistryOperations registryOperations = RegistryOperationsFactory.createInstance(JOYConstants.YARN_REGISTRY, conf); try { setupInitialRegistryPaths(); } catch (IOException e) { e.printStackTrace(); } registryOperations.start(); }
private static HAServiceState getHAState(YarnConfiguration yarnConf) throws Exception { HAServiceTarget haServiceTarget; int rpcTimeoutForChecks = yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, yarnConf.get(YarnConfiguration.RM_PRINCIPAL, "")); haServiceTarget = new RMHAServiceTarget(yarnConf); HAServiceProtocol proto = haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); HAServiceState haState = proto.getServiceStatus().getState(); return haState; }
public static boolean verifyStringsInNamedLogFiles( final String[] mustHave, final String fileName) { List<String> mustHaveList = Arrays.asList(mustHave); File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY)); if (!cwd.exists() || !cwd.isDirectory()) { return false;
private static HAServiceState getHAState(YarnConfiguration yarnConf) throws Exception { HAServiceTarget haServiceTarget; int rpcTimeoutForChecks = yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY, CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT); yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, yarnConf.get(YarnConfiguration.RM_PRINCIPAL, "")); haServiceTarget = new RMHAServiceTarget(yarnConf); HAServiceProtocol proto = haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks); HAServiceState haState = proto.getServiceStatus().getState(); return haState; }
File target = new File("../target" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY)); if (!target.mkdirs()) { LOG.warn("Error creating dirs to {}", target);
File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY)); Assert.assertTrue("Expecting directory " + cwd.getAbsolutePath() + " to exist", cwd.exists()); Assert.assertTrue("Expecting directory " + cwd.getAbsolutePath() + " to be a directory", cwd.isDirectory());
LOG.info("Starting up MiniYARNCluster"); if (yarnCluster == null) { final String testName = conf.get(YarnTestBase.TEST_CLUSTER_NAME_KEY); yarnCluster = new MiniYARNCluster( testName == null ? "YarnTest_" + UUID.randomUUID() : testName,
Credentials creds = new Credentials(); String tokRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
static boolean hasHttpsPolicy(Configuration conf) { YarnConfiguration yarnConf = new YarnConfiguration(conf); return (HttpConfig.Policy.HTTPS_ONLY == HttpConfig.Policy.fromString(yarnConf .get(YarnConfiguration.YARN_HTTP_POLICY_KEY, YarnConfiguration.YARN_HTTP_POLICY_DEFAULT))); }
protected String findRedirectUrl() throws ServletException { String addr; if (proxyUriBases.size() == 1) { // external proxy or not RM HA addr = proxyUriBases.values().iterator().next(); } else { // RM HA YarnConfiguration conf = new YarnConfiguration(); String activeRMId = RMHAUtils.findActiveRMHAId(conf); String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS : YarnConfiguration.RM_WEBAPP_ADDRESS; String host = conf.get( HAUtil.addSuffix(addressPropertyPrefix, activeRMId)); addr = proxyUriBases.get(host); } if (addr == null) { throw new ServletException( "Could not determine the proxy server for redirection"); } return addr; } }
public TwillRunnerService startTwillRunner(YarnConfiguration yarnConfiguration) { String zkStr = dremioConfig.getString(DremioConfig.ZOOKEEPER_QUORUM); String clusterId = yarnConfiguration.get(YARN_CLUSTER_ID); Preconditions.checkNotNull(clusterId, "Cluster ID can not be null"); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr); TwillRunnerService previousOne = twillRunners.putIfAbsent(new ClusterId(clusterId), twillRunner); if (previousOne == null) { // start one we are planning to add - if it is already in collection it should be started twillRunner.start(); return twillRunner; } return previousOne; }
public AMRMProtocol getAMResourceManager() { if (amResourceManager != null) return amResourceManager; LOG.debug("Using configuration: " + conf); YarnConfiguration yarnConf = new YarnConfiguration(conf); YarnRPC rpc = YarnRPC.create(yarnConf); InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get( YarnConfiguration.RM_SCHEDULER_ADDRESS, YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS)); LOG.info("Connecting to the resource manager (scheduling) at " + rmAddress); amResourceManager = (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress, conf); return amResourceManager; }
public ClientRMProtocol getClientResourceManager() { if (clientResourceManager != null) return clientResourceManager; YarnConfiguration yarnConf = new YarnConfiguration(conf); YarnRPC rpc = YarnRPC.create(yarnConf); InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get( YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); LOG.info("Connecting to the resource manager (client) at " + rmAddress); clientResourceManager = (ClientRMProtocol) rpc.getProxy( ClientRMProtocol.class, rmAddress, conf); return clientResourceManager; }
public YARNBSPJob(HamaConfiguration conf) throws IOException { submitClient = new YARNBSPJobClient(conf); YarnConfiguration yarnConf = new YarnConfiguration(conf); this.rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get( YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS)); LOG.info("Connecting to ResourceManager at " + rmAddress); this.applicationsManager = ((ApplicationClientProtocol) rpc.getProxy( ApplicationClientProtocol.class, rmAddress, conf)); }
@Test public void testDistroDefaultsOverwrite() throws Exception { assumeNonMaprProfile(); YarnController controller = Mockito.mock(YarnController.class); YarnService yarnService = new YarnService(new TestListener(), controller, Mockito.mock(NodeProvider.class)); Cluster myCluster = createCluster(); List<Property> props = myCluster.getClusterConfig().getSubPropertyList(); props.add(new Property(YarnDefaultsConfigurator.SPILL_PATH, "/abc/bcd")); props.add(new Property(YarnDefaultsConfigurator.JAVA_LOGIN, "/abc/bcd/login.conf")); myCluster.getClusterConfig().setDistroType(DistroType.HDP).setIsSecure(true); YarnConfiguration myYarnConfig = new YarnConfiguration(); yarnService.updateYarnConfiguration(myCluster, myYarnConfig); assertEquals("/abc/bcd/login.conf", myYarnConfig.get(YarnDefaultsConfigurator.JAVA_LOGIN)); assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT)); assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_CLIENT_CONFIG)); assertNull(myYarnConfig.get(YarnDefaultsConfigurator.ZK_SASL_PROVIDER)); assertEquals("/abc/bcd", myYarnConfig.get(YarnDefaultsConfigurator.SPILL_PATH)); }