private YarnClusterConfiguration prepareYarnCluster() throws IOException, URISyntaxException { yarnCluster.init(yarnConf); yarnCluster.start(); yarnConf.set(RM_ADDRESS, yarnCluster.getResourceManager().getConfig().get(RM_ADDRESS)); File yarnSite = new File(workDir, "yarn-site.xml"); try (PrintWriter pw = new PrintWriter(new FileWriter(yarnSite))) { yarnConf.writeXml(pw); } Path flinkUberJar = new Path(new File(workDir, "flink.jar").toURI()); Path flinkConfYaml = new Path(new File(workDir, "flink-conf.yaml").toURI()); @SuppressWarnings("ConstantConditions") Path log4jPath = new Path(Thread.currentThread().getContextClassLoader().getResource("log4j.properties").toURI()); Set<Path> resourcesToLocalize = new HashSet<>(Arrays.asList(flinkUberJar, flinkConfYaml, log4jPath)); String home = workDir.toURI().toString(); return new YarnClusterConfiguration( yarnConf, home, flinkUberJar, resourcesToLocalize, systemJars(yarnSite)); }
public RMApp submitApp(int masterMemory, String name, String user, boolean unmanaged) throws Exception { return submitApp(masterMemory, name, user, null, unmanaged, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); }
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, String queue, boolean waitForAccepted) throws Exception { return submitApp(masterMemory, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, waitForAccepted); }
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, String queue) throws Exception { return submitApp(masterMemory, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); }
public RMApp submitApp(int masterMemory, String queue) throws Exception { return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser().getShortUserName(), null, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); }
public RMApp submitApp(Resource resource, String name, String user, Map<ApplicationAccessType, String> acls, boolean unManaged, String queue) throws Exception { return submitApp(resource, name, user, acls, unManaged, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, null); }
public RMApp submitApp(int masterMemory, String name, String user, boolean unmanaged) throws Exception { return submitApp(masterMemory, name, user, null, unmanaged, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); }
public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, logAggregationContext, true); }
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls) throws Exception { return submitApp(masterMemory, name, user, acls, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); }
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, String queue) throws Exception { return submitApp(masterMemory, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null); }
public RMApp submitApp(Resource resource, String name, String user, Map<ApplicationAccessType, String> acls, String queue) throws Exception { return submitApp(resource, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, null); }
public RMApp submitApp(int masterMemory, Priority priority, Map<ApplicationTimeoutType, Long> applicationTimeouts) throws Exception { Resource resource = Resource.newInstance(masterMemory, 0); return submitApp( resource, "", UserGroupInformation.getCurrentUser().getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, priority, null, applicationTimeouts, null); }
public RMApp submitApp(int masterMemory, Priority priority, Credentials cred, ByteBuffer tokensConf) throws Exception { Resource resource = Resource.newInstance(masterMemory, 0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, false, false, null, 0, null, true, priority, null, null, tokensConf); }
public RMApp submitApp(int masterMemory, Priority priority) throws Exception { Resource resource = Resource.newInstance(masterMemory, 0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, priority); }
public RMApp submitApp(List<ResourceRequest> amResourceRequests) throws Exception { return submitApp(amResourceRequests, "app1", "user", null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, amResourceRequests.get(0).getPriority(), amResourceRequests.get(0).getNodeLabelExpression(), null, null); }
public RMApp submitApp(List<ResourceRequest> amResourceRequests, String appNodeLabel) throws Exception { return submitApp(amResourceRequests, "app1", "user", null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, amResourceRequests.get(0).getPriority(), amResourceRequests.get(0).getNodeLabelExpression(), null, null, null, appNodeLabel); }
public RMApp submitApp(int masterMemory, LogAggregationContext logAggregationContext) throws Exception { Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, logAggregationContext, true, priority); }
public RMApp submitApp(int masterMemory, Set<String> appTags) throws Exception { Resource resource = Resource.newInstance(masterMemory, 0); ResourceRequest amResourceRequest = ResourceRequest.newInstance( Priority.newInstance(0), ResourceRequest.ANY, resource, 1); return submitApp(Collections.singletonList(amResourceRequest), "", UserGroupInformation.getCurrentUser().getShortUserName(), null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, Priority.newInstance(0), null, null, null, appTags); }
public RMApp submitApp(Credentials cred, ByteBuffer tokensConf) throws Exception { return submitApp(Resource.newInstance(200, 1), "app1", "user", null, false, null, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true, false, false, null, 0, null, true, Priority.newInstance(0), null, null, tokensConf); }
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, String queue, String amLabel) throws Exception { Resource resource = Records.newRecord(Resource.class); resource.setMemorySize(masterMemory); Priority priority = Priority.newInstance(0); return submitApp(resource, name, user, acls, false, queue, super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, false, false, null, 0, null, true, priority, amLabel, null, null); }