private void addGroup(Pipeline p, StageGroup g) { if(p.hasGroup(g.getName())) { p.getGroup(g.getName()).setProperties(g.toPropertiesMap()); p.getGroup(g.getName()).setPropertiesModifiedDate(g.getPropertiesModifiedDate()); } else { p.addGroup(g); } }
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (getClass() != obj.getClass()) { return false; } StageGroup g = (StageGroup) obj; if(g.getSize() == stages.size()) { for(Stage s : g.getStages()) { if(!hasStage(s.getName()) || !getStage(s.getName()).equals(s)) { return false; } } if((getPropertiesModifiedDate()==null) != (g.getPropertiesModifiedDate()==null)) { return false; } else if (getPropertiesModifiedDate()==null && g.getPropertiesModifiedDate() == null) { return true; } return getPropertiesModifiedDate().equals(g.getPropertiesModifiedDate()); } return false; }
private Set<String> getChangedGroups(Pipeline newPipeline) { HashSet<String> list = new HashSet<String>(); for(StageGroup group : pipeline.getStageGroups()) { if(!newPipeline.hasGroup(group.getName()) || !group.equals(newPipeline.getGroup(group.getName()))) { list.add(group.getName()); } } return list; }
obj.put(MongoPipelineReader.NAME_KEY, group.getName()); if(group.isPropertiesChanged()) { props.put(MongoPipelineReader.PROPERTIES_DATE_SUBKEY, new Date()); props.put(MongoPipelineReader.PROPERTIES_DATE_SUBKEY, group.getPropertiesModifiedDate()); props.put(MongoPipelineReader.PROPERTIES_MAP_SUBKEY, group.toPropertiesMap()); for(Stage s : group.getStages()) { if(s.getMode() == Mode.ACTIVE) { active = true;
public static Stage getStage(Pipeline c, Object id, String stageName, String className, String ... afterStage) throws Exception { Stage s = new Stage(stageName, new DatabaseFile()); s.getDatabaseFile().setId(id); StageGroup g = new StageGroup(stageName); c.addGroup(g); g.addStage(s); Map<String, Object> props = new HashMap<String, Object>(); props.put("stageClass", "com.findwise.hydra."+className); if(afterStage!=null && afterStage.length>0) { props.put("queryOptions", new String[]{"touched("+afterStage[0]+",true)"}); } s.setProperties(props); return s; }
for (DatabaseFile df : stageGroup.getDatabaseFiles()) { File f = new File(targetDirectory, df.getFilename()); files.add(f); setParameters(stageGroup.toPropertiesMap()); if (stageGroup.getSize() == 1) { setParameters(stageGroup.getStages().iterator().next().getProperties());
public StageRunner(StageGroup stageGroup, File baseDirectory, int pipelinePort, boolean performanceLogging, int loggingPort, ShutdownHandler shutdownHandler) { this.stageGroup = stageGroup; this.baseDirectory = baseDirectory; this.targetDirectory = new File(baseDirectory, stageGroup.getName()); this.pipelinePort = pipelinePort; this.performanceLogging = performanceLogging; this.loggingPort = loggingPort; this.shutdownHandler = shutdownHandler; timesStarted = 0; }
@SuppressWarnings("unchecked") private StageGroup getGroup(DBObject dbo) { ((DBObject)dbo.get(PROPERTIES_KEY)).get(PROPERTIES_MAP_SUBKEY); StageGroup sg = new StageGroup((String)dbo.get(NAME_KEY), ((DBObject)((DBObject)dbo.get(PROPERTIES_KEY)).get(PROPERTIES_MAP_SUBKEY)).toMap()); sg.setPropertiesModifiedDate(sg.getPropertiesModifiedDate()); return sg; }
public static void removeStage(MongoConnector mdc, CommandLine cmd) throws IOException { if(!cmd.hasOption("n")) { System.out.println("No stage name specified\n"); printUsage(cmd); return; } String name = cmd.getOptionValue("n"); Pipeline pipeline = mdc.getPipelineReader().getPipeline(); if(pipeline.getStage(name) == null) { System.out.println("Specified stage '"+name+"' did not exist\n"); return; } boolean found = false; for(StageGroup g : pipeline.getStageGroups()) { if(g.hasStage(name)) { g.removeStage(name); found = true; break; } } if(found) { mdc.getPipelineWriter().write(pipeline); System.out.println("Successfully removed stage '"+name+"'"); } else { System.out.println("Unable to delete '"+name+"'. Stage did not exist."); } }
public void run() { started = true; if (!prepared) { logger.error("The StageRunner was not prepared prior to being started. Aborting!"); return; } if (stageGroup.isEmpty()) { logger.info("Stage group " + stageGroup.getName() + " has no stages, and can not be started."); return; } do { logger.info("Starting stage group " + stageGroup.getName() + ". Times started so far: " + timesStarted); timesStarted++; boolean cleanShutdown = runGroup(); if (cleanShutdown) { return; } if (!hasQueried()) { logger.error("The stage group " + stageGroup.getName() + " did not start. It will not be restarted until configuration changes."); return; } } while ((timesToRetry == -1 || timesToRetry >= timesStarted) && !shutdownHandler.isShuttingDown()); logger.error("Stage group " + stageGroup.getName() + " has failed and cannot be restarted. "); }
public StageGroup getGroupForStage(String name) { for (StageGroup g : getStageGroups()) { if (g.hasStage(name)) { return g; } } return null; }
public Set<Stage> getStages() { HashSet<Stage> stages = new HashSet<Stage>(); for(StageGroup g : getStageGroups()) { stages.addAll(g.getStages()); } return stages; }
public StageGroup(String name, Map<String, Object> propertiesMap) { this(name); setProperties(propertiesMap); }
StageGroup g = new StageGroup(s.getName()); g.addStage(s); pipeline.addGroup(g); mdc.getPipelineWriter().write(pipeline);
public void addGroup(StageGroup g) { stageGroups.put(g.getName(), g); }
if(g.hasStage(s.getName())) { g.removeStage(s.getName());
public StageRunner getRunnerForStage(String stageName) { for(StageRunner runner : runnerMap.values()) { if(runner.getStageGroup().hasStage(stageName)) { return runner; } } return null; }
private Pipeline getPipeline(Stage.Mode mode) { Pipeline p = new Pipeline(); DBCursor cursor = stages.find(new BasicDBObject(ACTIVE_KEY, mode.toString())); while(cursor.hasNext()) { DBObject obj = cursor.next(); if(!obj.containsField(TYPE_KEY) || STAGE_TYPE.equals(obj.get(TYPE_KEY))) { Stage s = getStage(obj); if(!p.hasGroup(getGroupName(obj))) { p.addGroup(new StageGroup(getGroupName(obj))); } p.getGroup(getGroupName(obj)).addStage(s); } else { addGroup(p, getGroup(obj)); } } return p; }
public void addRunner(StageRunner runner) { runnerMap.put(runner.getStageGroup().getName(), runner); }