.setMemory(memoryOnHeapInt+memoryOffHeapInt, ResourceSpecification.SizeUnit.MEGA) .setInstances(containerCountInt).build()) .withLocalFiles() .add(YARN_BUNDLED_JAR_NAME, yarnBundledJarPath.toUri(), false);
/** * Adds localize resources for the given service. */ private Builder.RunnableSetter addResources(String service, Builder.RuntimeSpecificationAdder adder) { Map<String, LocalizeResource> localizeResources = runnableLocalizeResources.get(service); Iterator<Map.Entry<String, LocalizeResource>> iterator = localizeResources.entrySet().iterator(); TwillSpecification.Builder.MoreFile moreFile = null; while (iterator.hasNext()) { Map.Entry<String, LocalizeResource> entry = iterator.next(); if (moreFile == null) { moreFile = adder.withLocalFiles().add(entry.getKey(), entry.getValue().getURI(), entry.getValue().isArchive()); } else { moreFile = moreFile.add(entry.getKey(), entry.getValue().getURI(), entry.getValue().isArchive()); } } return moreFile == null ? adder.noLocalFiles() : moreFile.apply(); }
@Override public RuntimeSpecificationAdder add(String name, TwillRunnable runnable, final ResourceSpecification resourceSpec) { final TwillRunnableSpecification spec = new DefaultTwillRunnableSpecification( runnable.getClass().getName(), name, runnable.configure().getConfigs()); return new RuntimeSpecificationAdder(new LocalFileCompleter() { @Override public RunnableSetter complete(Collection<LocalFile> files) { runnables.put(spec.getName(), new DefaultRuntimeSpecification(spec.getName(), spec, resourceSpec, files)); return RunnableSetter.this; } }); }
.add(org.apache.fluo.cluster.runnable.OracleRunnable.ORACLE_NAME, new org.apache.fluo.cluster.runnable.OracleRunnable(), oracleResources) .withLocalFiles(); RunnableSetter runnableSetter = addConfigFiles(fileAdder).apply(); .add(org.apache.fluo.cluster.runnable.WorkerRunnable.WORKER_NAME, new org.apache.fluo.cluster.runnable.WorkerRunnable(), workerResources) .withLocalFiles(); runnableSetter = addConfigFiles(fileAdder).apply();
.add(org.apache.fluo.cluster.runnable.OracleRunnable.ORACLE_NAME, new org.apache.fluo.cluster.runnable.OracleRunnable(), oracleResources) .withLocalFiles(); RunnableSetter runnableSetter = addConfigFiles(fileAdder).apply(); .add(org.apache.fluo.cluster.runnable.WorkerRunnable.WORKER_NAME, new org.apache.fluo.cluster.runnable.WorkerRunnable(), workerResources) .withLocalFiles(); runnableSetter = addConfigFiles(fileAdder).apply();
runnableSetter = moreRunnable .add(flowletName, new FlowletTwillRunnable(flowletName, "hConf.xml", "cConf.xml"), resourceSpec) .withLocalFiles().add(programName, programLocation.toURI()) .add("hConf.xml", hConfig.toURI()) .add("cConf.xml", cConfig.toURI()).apply(); .withLocalFiles() .add("cConf.xml", cConfig.toURI()) .add("hConf.xml", hConfig.toURI())
.withLocalFiles(); RunnableSetter runnableSetter = addConfigFiles(fileAdder).apply(); .withLocalFiles(); runnableSetter = addConfigFiles(fileAdder).apply();
@Override public TwillSpecification configure() { ResourceSpecification largeRes = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(1024, ResourceSpecification.SizeUnit.MEGA) .build(); ResourceSpecification smallRes = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .build(); return TwillSpecification.Builder.with() .setName("SleepApp") .withRunnable() .add("sleep1", new SleepRunnable(12345), largeRes).noLocalFiles() .add("sleep2", new SleepRunnable(12346), smallRes).noLocalFiles() .withOrder() .begin("sleep1") .nextWhenStarted("sleep2") .build(); } }
@Test public void testOrder() { TwillSpecification spec = TwillSpecification.Builder.with() .setName("Testing") .withRunnable() .add("r1", new DummyRunnable()).noLocalFiles() .add("r2", new DummyRunnable()).noLocalFiles() .add("r3", new DummyRunnable()).noLocalFiles() .add("r4", new DummyRunnable()).noLocalFiles() .withOrder().begin("r1", "r2").nextWhenStarted("r3") .build(); Assert.assertEquals(4, spec.getRunnables().size()); List<TwillSpecification.Order> orders = spec.getOrders(); Assert.assertEquals(3, orders.size()); Assert.assertEquals(ImmutableSet.of("r1", "r2"), orders.get(0).getNames()); Assert.assertEquals(ImmutableSet.of("r3"), orders.get(1).getNames()); Assert.assertEquals(ImmutableSet.of("r4"), orders.get(2).getNames()); } }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("ResourceApplication") .withRunnable() .add("echo1", new EchoServer(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(256, ResourceSpecification.SizeUnit.MEGA) .setInstances(2).build()).noLocalFiles() .add("echo2", new EchoServer(), ResourceSpecification.Builder.with() .setVirtualCores(2) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(1).build()).noLocalFiles() .anyOrder() .build(); } }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("PlacementPolicyApplication") .withRunnable() .add("hostRunnable", new EchoServer(), resource).noLocalFiles() .add("hostRackRunnable", new EchoServer(), resource).noLocalFiles() .add("distributedRunnable", new EchoServer(), twoInstancesResource).noLocalFiles() .withPlacementPolicy() .add(Hosts.of(nodeReports.get(0).getHttpAddress()), "hostRunnable") .add(Hosts.of(nodeReports.get(1).getHttpAddress()), Racks.of("/default-rack"), "hostRackRunnable") .add(TwillSpecification.PlacementPolicy.Type.DISTRIBUTED, "distributedRunnable") .anyOrder() .build(); } }
@Override public TwillSpecification configure() { // Make the runnable request for container smaller than 128MB (the allocation minimum) ResourceSpecification res = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(16, ResourceSpecification.SizeUnit.MEGA) .build(); return TwillSpecification.Builder.with() .setName("MaxHeapApp") .withRunnable() .add("sleep", new MaxHeapRunnable(12345), res).noLocalFiles() .add("sleep2", new MaxHeapRunnable(23456), res).noLocalFiles() .anyOrder() .build(); } }
@Test public void testAnyOrder() { TwillSpecification spec = TwillSpecification.Builder.with() .setName("Testing") .withRunnable() .add("r1", new DummyRunnable()).noLocalFiles() .add("r2", new DummyRunnable()).noLocalFiles() .add("r3", new DummyRunnable()).noLocalFiles() .anyOrder() .build(); Assert.assertEquals(3, spec.getRunnables().size()); List<TwillSpecification.Order> orders = spec.getOrders(); Assert.assertEquals(1, orders.size()); Assert.assertEquals(ImmutableSet.of("r1", "r2", "r3"), orders.get(0).getNames()); }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName(APPLICATION_NAME) .withRunnable() .add(PeriodicNotificationTwillRunnable.TWILL_RUNNABLE_NAME, new PeriodicNotificationTwillRunnable(), ResourceSpecification.Builder.with() .setVirtualCores(2) .setMemory(2, SizeUnit.GIGA) .setInstances(1) .build()) .withLocalFiles() .add(PeriodicNotificationTwillRunnable.CONFIG_FILE_NAME, configFile) .apply() .anyOrder() .build(); }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("BadApplication") .withRunnable() .add("Hermione", new EchoServer(), resource).noLocalFiles() .add("Harry", new EchoServer(), resource).noLocalFiles() .add("Ron", new EchoServer(), resource).noLocalFiles() .withPlacementPolicy() .add(TwillSpecification.PlacementPolicy.Type.DEFAULT, "Hermione", "Harry") .add(TwillSpecification.PlacementPolicy.Type.DISTRIBUTED, "Hermione", "Ron") .anyOrder() .build(); } }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("FaultyApplication") .withRunnable() .add("Hermione", new EchoServer(), resource).noLocalFiles() .add("Harry", new EchoServer(), resource).noLocalFiles() .add("Ron", new EchoServer(), resource).noLocalFiles() .withPlacementPolicy() .add(TwillSpecification.PlacementPolicy.Type.DEFAULT, "Hermione", "Ron") .add(TwillSpecification.PlacementPolicy.Type.DISTRIBUTED, "Draco", "Harry") .anyOrder() .build(); } }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("DistributedApplication") .withRunnable() .add("Alice", new EchoServer(), twoInstancesResource).noLocalFiles() .add("Bob", new EchoServer(), resource).noLocalFiles() .add("Eve", new EchoServer(), resource).noLocalFiles() .withPlacementPolicy() .add(TwillSpecification.PlacementPolicy.Type.DISTRIBUTED, "Alice", "Bob") .anyOrder() .build(); } }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("DistributedApplication") .withRunnable() .add("Alice", new EchoServer(), resource).noLocalFiles() .add("Bob", new EchoServer(), resource).noLocalFiles() .add("Eve", new EchoServer(), resource).noLocalFiles() .withPlacementPolicy() .add(TwillSpecification.PlacementPolicy.Type.DISTRIBUTED, "Alice", "Bob") .anyOrder() .build(); } }
/** * Request localization of the program jar and all other files. */ private Builder.RunnableSetter localizeFiles(Map<String, LocalizeResource> localizeResources, Builder.RuntimeSpecificationAdder builder) { Builder.LocalFileAdder fileAdder; Builder.MoreFile moreFile = null; for (Map.Entry<String, LocalizeResource> entry : localizeResources.entrySet()) { LOG.debug("Localizing file for {}: {} {}", programRunId, entry.getKey(), entry.getValue()); fileAdder = (moreFile == null) ? builder.withLocalFiles() : moreFile; moreFile = fileAdder.add(entry.getKey(), entry.getValue().getURI(), entry.getValue().isArchive()); } return moreFile == null ? builder.noLocalFiles() : moreFile.apply(); }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("TimeoutApplication") .withRunnable() .add(new TimeoutRunnable(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(8, ResourceSpecification.SizeUnit.GIGA).build()) .noLocalFiles() .anyOrder() .withEventHandler(new Handler(parentFolderPath)) .build(); } }