@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
@BeforeClass public static void setup() { YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-per-job"); startYARNWithConfig(YARN_CONFIGURATION); }
private void setYarnDefaults(Cluster cluster, YarnConfiguration yarnConfiguration) { DistroType dType = cluster.getClusterConfig().getDistroType(); Map<String, String> additionalProps = defaultsConfigurator.getDistroTypeDefaultsConfigurator(dType, cluster.getClusterConfig().getIsSecure()).getAllDefaults(); for (Map.Entry<String,String> entry : additionalProps.entrySet()) { yarnConfiguration.set(entry.getKey(), entry.getValue()); } // Fix for DX-12202: on hdp 2.8.3, timeline service might be enabled by default, hence disabling it here yarnConfiguration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false); }
@Before public void setup() { conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); conf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE, YarnConfiguration.DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE); conf.setClass(YarnConfiguration.RM_NODE_LABELS_PROVIDER_CONFIG, DummyRMNodeLabelsMappingProvider.class, RMNodeLabelsMappingProvider.class); }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = getConf(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0); DefaultMetricsSystem.setMiniClusterMode(true); }
clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
@Before public void setup() throws IOException { conf = getConf(); Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); UserGroupInformation.setConfiguration(conf); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); rmAddr = new InetSocketAddress("localhost", 8032); Assert.assertTrue(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS > 1); }
private YarnClusterConfiguration prepareYarnCluster() throws IOException, URISyntaxException { yarnCluster.init(yarnConf); yarnCluster.start(); yarnConf.set(RM_ADDRESS, yarnCluster.getResourceManager().getConfig().get(RM_ADDRESS)); File yarnSite = new File(workDir, "yarn-site.xml"); try (PrintWriter pw = new PrintWriter(new FileWriter(yarnSite))) { yarnConf.writeXml(pw); } Path flinkUberJar = new Path(new File(workDir, "flink.jar").toURI()); Path flinkConfYaml = new Path(new File(workDir, "flink-conf.yaml").toURI()); @SuppressWarnings("ConstantConditions") Path log4jPath = new Path(Thread.currentThread().getContextClassLoader().getResource("log4j.properties").toURI()); Set<Path> resourcesToLocalize = new HashSet<>(Arrays.asList(flinkUberJar, flinkConfYaml, log4jPath)); String home = workDir.toURI().toString(); return new YarnClusterConfiguration( yarnConf, home, flinkUberJar, resourcesToLocalize, systemJars(yarnSite)); }
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); storageDir = FOLDER.newFolder().getAbsolutePath(); // startYARNWithConfig should be implemented by subclass YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096); startYARNWithConfig(YARN_CONFIGURATION); }
@Test public void testGetMasterAddress() { YarnConfiguration conf = new YarnConfiguration(); // Trying invalid master address for classic conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); conf.set(MRConfig.MASTER_ADDRESS, "local:invalid"); // should throw an exception for invalid value try { Master.getMasterAddress(conf); fail("Should not reach here as there is a bad master address"); } catch (Exception e) { // Expected } // Change master address to a valid value conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042"); String masterHostname = Master.getMasterAddress(conf); assertEquals(masterHostname, "bar.com"); } }
YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo-secured");
@Override public synchronized void performFailover(T currentProxy) { currentProxyIndex = (currentProxyIndex + 1) % rmServiceIds.length; conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); LOG.info("Failing over to " + rmServiceIds[currentProxyIndex]); }
@Override public synchronized void performFailover(T currentProxy) { currentProxyIndex = (currentProxyIndex + 1) % rmServiceIds.length; conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); LOG.info("Failing over to " + rmServiceIds[currentProxyIndex]); }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); Assert.assertTrue(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS > 1); }
@Before public void setup() throws IOException { FileUtil.fullyDelete(TEST_DIR); conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_LEVELDB_STORE_PATH, TEST_DIR.toString()); }
@Before public void setup() { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); ExitUtil.disableSystemExit(); testConf = new YarnConfiguration(); testConf .set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); UserGroupInformation.setLoginUser(null); UserGroupInformation.setConfiguration(testConf); }
@Test public void testQueueMappingTrimSpaces() throws IOException { // space trimming conf.set(CapacitySchedulerConfiguration.QUEUE_MAPPING, " u : a : " + Q1); cs.reinitialize(conf, null); checkQMapping(new QueueMapping(MappingType.USER, "a", Q1)); }
private MockRM startRM(String rmId, HAServiceState state) throws Exception{ YarnConfiguration yarnConf = new YarnConfiguration(conf); yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); MockRM rm = new MockRM(yarnConf, true); rm.init(yarnConf); rm.start(); waitFor(rm, state); return rm; }
@Test(timeout = 20000) public void testSetZKAcl() { TestZKClient zkClientTester = new TestZKClient(); YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_ZK_ACL, "world:anyone:rwca"); try { zkClientTester.store.zkClient.delete(zkClientTester.store .znodeWorkingPath, -1); fail("Shouldn't be able to delete path"); } catch (Exception e) {/* expected behavior */ } }