@Override public JsonElement serialize(FlowletSpecification src, Type typeOfSrc, JsonSerializationContext context) { JsonObject jsonObj = new JsonObject(); jsonObj.add("className", new JsonPrimitive(src.getClassName())); jsonObj.add("name", new JsonPrimitive(src.getName())); jsonObj.add("description", new JsonPrimitive(src.getDescription())); jsonObj.add("failurePolicy", new JsonPrimitive(src.getFailurePolicy().name())); jsonObj.add("properties", serializeMap(src.getProperties(), context, String.class)); jsonObj.add("resources", context.serialize(src.getResources(), new TypeToken<ResourceSpecification>() { }.getType())); jsonObj.add("maxInstances", new JsonPrimitive(src.getMaxInstances())); return jsonObj; }
int flowletMaxInstances = flowletSpecification.getMaxInstances(); Preconditions.checkArgument(newInstanceCount <= flowletMaxInstances, "Flowlet %s can have a maximum of %s instances", flowletSpecification.getName(), flowletMaxInstances);
@Override public MoreFlowlet add(String name, Flowlet flowlet, int instances) { Preconditions.checkArgument(flowlet != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL)); FlowletDefinition flowletDef = new FlowletDefinition(name, flowlet, instances); String flowletName = flowletDef.getFlowletSpec().getName(); Preconditions.checkArgument(instances > 0, String.format(UserMessages.getMessage(UserErrors.INVALID_INSTANCES), flowletName, instances)); Preconditions.checkArgument(!flowlets.containsKey(flowletName), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_EXISTS), flowletName); flowlets.put(flowletName, flowletDef); return this; }
Preconditions.checkNotNull(flowletDef, "Definition missing for flowlet \"%s\"", flowletName); Class<?> clz = Class.forName(flowletDef.getFlowletSpec().getClassName(), true, program.getClassLoader()); Preconditions.checkArgument(Flowlet.class.isAssignableFrom(clz), "%s is not a Flowlet.", clz); new PropertyFieldSetter(flowletDef.getFlowletSpec().getProperties()), new MetricsFieldSetter(flowletContext.getMetrics()), new OutputEmitterFieldSetter(outputEmitterFactory(flowletContext, flowletName,
private Callback createCallback(Flowlet flowlet, FlowletSpecification flowletSpec) { if (flowlet instanceof Callback) { return (Callback) flowlet; } final FailurePolicy failurePolicy = flowletSpec.getFailurePolicy(); return new Callback() { @Override public void onSuccess(Object input, InputContext inputContext) { // No-op } @Override public FailurePolicy onFailure(Object input, InputContext inputContext, FailureReason reason) { return failurePolicy; } }; }
void update(String flowletId, int newInstanceCount, int oldInstanceCount) throws Exception { FlowletDefinition flowletDefinition = program.getSpecification().getFlowlets().get(flowletId); int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances(); Preconditions.checkArgument(newInstanceCount <= maxInstances, "Flowlet %s can have a maximum of %s instances", flowletId, maxInstances); waitForInstances(flowletId, oldInstanceCount); twillController.sendCommand(flowletId, ProgramCommands.SUSPEND).get(); FlowUtils.reconfigure(consumerQueues.get(flowletId), FlowUtils.generateConsumerGroupId(program, flowletId), newInstanceCount, queueAdmin); twillController.changeInstances(flowletId, newInstanceCount).get(); twillController.sendCommand(flowletId, ProgramCommands.RESUME).get(); }
FlowletSpecification flowletSpec = flowletDefinition.getFlowletSpec(); ResourceSpecification resourceSpec = ResourceSpecification.Builder.with() .setVirtualCores(flowletSpec.getResources().getVirtualCores()) .setMemory(flowletSpec.getResources().getMemoryMB(), ResourceSpecification.SizeUnit.MEGA) .setInstances(flowletDefinition.getInstances()) .build();
@Override public ProgramController run(Program program, ProgramOptions options) { // Extract and verify parameters FlowSpecification flowSpec = program.getSpecification(); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported."); Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName()); for (FlowletDefinition flowletDefinition : flowSpec.getFlowlets().values()) { int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances(); Preconditions.checkArgument(flowletDefinition.getInstances() <= maxInstances, "Flowlet %s can have a maximum of %s instances", flowletDefinition.getFlowletSpec().getName(), maxInstances); } try { // Launch flowlet program runners RunId runId = RunIds.generate(); programOptions.put(runId, options); Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, flowSpec, queueAdmin); final Table<String, Integer, ProgramController> flowlets = createFlowlets(program, runId, flowSpec); return new FlowProgramController(flowlets, runId, program, flowSpec, consumerQueues, discoveryServiceClient); } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public MoreConnect to(String flowlet) { Preconditions.checkArgument(flowlet != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL)); Preconditions.checkArgument(flowlets.containsKey(flowlet), UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NAME), flowlet); FlowletConnection.Type sourceType; String sourceName; sourceType = FlowletConnection.Type.FLOWLET; sourceName = fromFlowlet.getFlowletSpec().getName(); connections.add(new FlowletConnection(sourceType, sourceName, flowlet)); return this; }
@Override public FailurePolicy onFailure(@Nullable Object input, @Nullable InputContext inputContext, FailureReason reason) { // Return the policy as specified in the spec return flowletContext.getSpecification().getFailurePolicy(); }
FlowletDefinition(String flowletName, Flowlet flowlet, int instances) { FlowletSpecification flowletSpec = flowlet.configure(); this.instances = instances; Map<String, Set<Type>> inputTypes = Maps.newHashMap(); Map<String, Set<Type>> outputTypes = Maps.newHashMap(); Map<String, String> properties = Maps.newHashMap(flowletSpec.getProperties()); Reflections.visit(flowlet, TypeToken.of(flowlet.getClass()), new PropertyFieldExtractor(properties), new OutputEmitterFieldExtractor(outputTypes), new ProcessMethodExtractor(inputTypes)); this.inputTypes = immutableCopyOf(inputTypes); this.outputTypes = immutableCopyOf(outputTypes); this.flowletSpec = new DefaultFlowletSpecification(flowlet.getClass().getName(), flowletName == null ? flowletSpec.getName() : flowletName, flowletSpec.getDescription(), flowletSpec.getFailurePolicy(), properties, flowletSpec.getResources(), flowletSpec.getMaxInstances()); }
@Override protected ProgramController launch(Program program, ProgramOptions options, File hConfFile, File cConfFile, ApplicationLauncher launcher) { // Extract and verify parameters FlowSpecification flowSpec = program.getSpecification(); ProgramType processorType = program.getType(); Preconditions.checkNotNull(processorType, "Missing processor type."); Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported."); try { Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName()); for (FlowletDefinition flowletDefinition : flowSpec.getFlowlets().values()) { int maxInstances = flowletDefinition.getFlowletSpec().getMaxInstances(); Preconditions.checkArgument(flowletDefinition.getInstances() <= maxInstances, "Flowlet %s can have a maximum of %s instances", flowletDefinition.getFlowletSpec().getName(), maxInstances); } LOG.info("Configuring flowlets queues"); Multimap<String, QueueName> flowletQueues = FlowUtils.configureQueue(program, flowSpec, queueAdmin); // Launch flowlet program runners LOG.info("Launching distributed flow: " + program.getName() + ":" + flowSpec.getName()); TwillController controller = launcher.launch(new FlowTwillApplication(program, flowSpec, hConfFile, cConfFile, eventHandler)); DistributedFlowletInstanceUpdater instanceUpdater = new DistributedFlowletInstanceUpdater(program, controller, queueAdmin, flowletQueues); return new FlowTwillProgramController(program.getName(), controller, instanceUpdater).startListen(); } catch (Exception e) { throw Throwables.propagate(e); } }
@Override public ConnectTo from(Flowlet flowlet) { Preconditions.checkArgument(flowlet != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL)); return from(flowlet.configure().getName()); }
@Override public MoreConnect to(Flowlet flowlet) { Preconditions.checkArgument(flowlet != null, UserMessages.getMessage(UserErrors.INVALID_FLOWLET_NULL)); return to(flowlet.configure().getName()); }