private void addMissingGroups(Pipeline newPipeline) { for(StageGroup group : newPipeline.getStageGroups()) { if(!pipeline.hasGroup(group.getName())) { pipeline.addGroup(group); if(attachFiles(group)) { sm.addRunner(new StageRunner(group, new File(namespace), port, conf.isPerformanceLogging(), conf.getLoggingPort(), shutdownHandler)); } else { logger.error("Was unable to start the stage group '"+group.getName()+"' due to missing libraries."); } } } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } Pipeline other = (Pipeline) obj; if(other.getStageGroups().size()==getStageGroups().size()) { for(StageGroup g : other.getStageGroups()) { if(!hasGroup(g.getName()) || !g.equals(getGroup(g.getName()))) { return false; } } return true; } return false; }
public boolean hasStage(String name) { return getStage(name) != null; } }
@Override public void write(Pipeline p) throws IOException { Pipeline old = reader.getPipeline(); for(Stage s : old.getStages()) { inactivate(s); } for(StageGroup g : p.getStageGroups()) { write(g); } }
private Pipeline getPipeline(Stage.Mode mode) { Pipeline p = new Pipeline(); DBCursor cursor = stages.find(new BasicDBObject(ACTIVE_KEY, mode.toString())); while(cursor.hasNext()) { DBObject obj = cursor.next(); if(!obj.containsField(TYPE_KEY) || STAGE_TYPE.equals(obj.get(TYPE_KEY))) { Stage s = getStage(obj); if(!p.hasGroup(getGroupName(obj))) { p.addGroup(new StageGroup(getGroupName(obj))); } p.getGroup(getGroupName(obj)).addStage(s); } else { addGroup(p, getGroup(obj)); } } return p; }
public static void removeStage(MongoConnector mdc, CommandLine cmd) throws IOException { if(!cmd.hasOption("n")) { System.out.println("No stage name specified\n"); printUsage(cmd); return; } String name = cmd.getOptionValue("n"); Pipeline pipeline = mdc.getPipelineReader().getPipeline(); if(pipeline.getStage(name) == null) { System.out.println("Specified stage '"+name+"' did not exist\n"); return; } boolean found = false; for(StageGroup g : pipeline.getStageGroups()) { if(g.hasStage(name)) { g.removeStage(name); found = true; break; } } if(found) { mdc.getPipelineWriter().write(pipeline); System.out.println("Successfully removed stage '"+name+"'"); } else { System.out.println("Unable to delete '"+name+"'. Stage did not exist."); } }
private void getStages(HttpRequest request, HttpResponse response, HttpContext context) { logger.trace("handleGetStages()"); String group = RESTTools.getParam(request, GroupStarter.GROUP_PARAM); Pipeline p = reader.getPipeline(); if(!p.hasGroup(group)) { p = reader.getDebugPipeline(); } if(p.hasGroup(group)) { HttpResponseWriter.printJson(response, p.getGroup(group).getStageNames()); return; } response.setStatusCode(HttpStatus.SC_NOT_FOUND); response.setEntity(new NStringEntity("", HttpResponseWriter.CONTENT_TYPE)); }
public StageGroup getGroupForStage(String name) { for (StageGroup g : getStageGroups()) { if (g.hasStage(name)) { return g; } } return null; }
public static Stage getStage(Pipeline c, Object id, String stageName, String className, String ... afterStage) throws Exception { Stage s = new Stage(stageName, new DatabaseFile()); s.getDatabaseFile().setId(id); StageGroup g = new StageGroup(stageName); c.addGroup(g); g.addStage(s); Map<String, Object> props = new HashMap<String, Object>(); props.put("stageClass", "com.findwise.hydra."+className); if(afterStage!=null && afterStage.length>0) { props.put("queryOptions", new String[]{"touched("+afterStage[0]+",true)"}); } s.setProperties(props); return s; }
private void getPropetries(HttpRequest request, HttpResponse response, HttpContext context) { logger.trace("handleGetProperties()"); String stage = RESTTools.getStage(request); logger.debug("Received getProperties()-request for stage: "+stage); if(stage==null) { HttpResponseWriter.printMissingParameter(response, HttpEndpointConstants.STAGE_PARAM); return; } Map<String, Object> map = new HashMap<String, Object>(); if(reader.getPipeline().hasStage(stage)) { map = reader.getPipeline().getStage(stage).getProperties(); } else if(reader.getDebugPipeline().hasStage(stage)){ map = reader.getDebugPipeline().getStage(stage).getProperties(); } HttpResponseWriter.printJson(response, map); }
Pipeline c = new Pipeline(); Stage s = getStage(c, basicId, "copyStage1", "stage.CopyStage"); Map<String, Object> map = s.getProperties();
public Stage getStage(String name) { for(Stage s : getStages()) { if(s.getName().equals(name)) { return s; } } return null; }
for(Stage s : pipeline.getStages()) { if(s.getDatabaseFile().getId().equals(cmd.getOptionValue("i"))) { stagesToDelete.add(s); for(StageGroup g : pipeline.getStageGroups()) { if(g.hasStage(s.getName())) { g.removeStage(s.getName());
public Set<Stage> getStages() { HashSet<Stage> stages = new HashSet<Stage>(); for(StageGroup g : getStageGroups()) { stages.addAll(g.getStages()); } return stages; }
StageGroup g = new StageGroup(s.getName()); g.addStage(s); pipeline.addGroup(g); mdc.getPipelineWriter().write(pipeline); System.out.println("Added stage " + name
coreConfiguration, caching, new Pipeline(), this);
private Set<String> getChangedGroups(Pipeline newPipeline) { HashSet<String> list = new HashSet<String>(); for(StageGroup group : pipeline.getStageGroups()) { if(!newPipeline.hasGroup(group.getName()) || !group.equals(newPipeline.getGroup(group.getName()))) { list.add(group.getName()); } } return list; }