@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; Stage s = (Stage) obj; if(!s.getName().equals(name) || !s.getPropertiesModifiedDate().equals(getPropertiesModifiedDate()) || !mode.equals(s.getMode())) { return false; } if(getDatabaseFile()==null) { return s.getDatabaseFile() == null; } return getDatabaseFile().equals(s.getDatabaseFile()); } }
public static Stage getStage(Pipeline c, Object id, String stageName, String className, String ... afterStage) throws Exception { Stage s = new Stage(stageName, new DatabaseFile()); s.getDatabaseFile().setId(id); StageGroup g = new StageGroup(stageName); c.addGroup(g); g.addStage(s); Map<String, Object> props = new HashMap<String, Object>(); props.put("stageClass", "com.findwise.hydra."+className); if(afterStage!=null && afterStage.length>0) { props.put("queryOptions", new String[]{"touched("+afterStage[0]+",true)"}); } s.setProperties(props); return s; }
public List<Stage> createStages(PipelineConfiguration pipelineConfig) { List<Stage> stages = new ArrayList<Stage>(); for (Map<String, Object> stageConfig : pipelineConfig) { String jarId = (String) stageConfig.get(JAR_ID); String stageName = (String) stageConfig.get(STAGE_NAME); DatabaseFile df = new DatabaseFile(); df.setId(jarId); Stage s = new Stage(stageName, df); s.setProperties(stageConfig); s.setMode(Stage.Mode.ACTIVE); stages.add(s); } return stages; } }
/** * May return null if and only if a referenced stage library was not found in the database. */ public Set<DatabaseFile> getDatabaseFiles() { Map<Object, DatabaseFile> files = new HashMap<Object, DatabaseFile>(); for(Stage s : stages.values()) { if(s.getDatabaseFile()==null) { logger.error("Stage group '"+s.getName()+"' is missing it's library file"); return null; } else if(!files.containsKey(s.getDatabaseFile().getId())) { files.put(s.getDatabaseFile().getId(), s.getDatabaseFile()); } } return new HashSet<DatabaseFile>(files.values()); }
private DBObject getStageDBObject(Stage s, String group) { BasicDBObject obj = new BasicDBObject(); obj.put(MongoPipelineReader.STAGE_KEY, s.getName()); obj.put(MongoPipelineReader.TYPE_KEY, MongoPipelineReader.STAGE_TYPE); BasicDBObject props = new BasicDBObject(); if(s.isPropertiesChanged()) { props.put(MongoPipelineReader.PROPERTIES_DATE_SUBKEY, new Date()); } else { props.put(MongoPipelineReader.PROPERTIES_DATE_SUBKEY, s.getPropertiesModifiedDate()); } props.put(MongoPipelineReader.PROPERTIES_MAP_SUBKEY, s.getProperties()); obj.put(MongoPipelineReader.PROPERTIES_KEY, props); obj.put(MongoPipelineReader.ACTIVE_KEY, s.getMode().toString()); obj.put(MongoPipelineReader.GROUP_KEY, group); if(s.getDatabaseFile()!=null) { obj.put(MongoPipelineReader.FILE_KEY, s.getDatabaseFile().getId()); } return obj; }
df.setId(libraryId); Stage s = new Stage(name, df); s.setProperties(map); if(debug) { s.setMode(Stage.Mode.DEBUG); } else { s.setMode(Stage.Mode.ACTIVE); StageGroup g = new StageGroup(s.getName()); g.addStage(s); pipeline.addGroup(g);
@SuppressWarnings("unchecked") private Stage getStage(DBObject dbo) { Stage stage = new Stage((String)dbo.get(STAGE_KEY), getFile(dbo.get(FILE_KEY))); DBObject props = (DBObject) dbo.get(PROPERTIES_KEY); stage.setMode(Mode.valueOf((String)dbo.get(ACTIVE_KEY))); stage.setPropertiesModifiedDate((Date)props.get(PROPERTIES_DATE_SUBKEY)); DBObject properties = (DBObject) props.get(PROPERTIES_MAP_SUBKEY); HashMap<String, Object> map = new HashMap<String, Object>(); if(properties!=null) { map.putAll(properties.toMap()); } stage.setProperties(map); return stage; }
public void addStage(Stage stage) { stages.put(stage.getName(), stage); }
Map<String, Object> map = s.getProperties(); map.put("map", getSingleMap("in", "out1")); s.setProperties(map); map = s.getProperties(); map.put("map", getSingleMap("out1", "out2")); s.setProperties(map); map = s.getProperties(); map.put("map", getSingleMap("out2", "out3")); s.setProperties(map); map = s.getProperties(); HashMap<String, String> fieldMap = new HashMap<String, String>(); fieldMap.put("out1", "out1_s"); map.put("fieldMappings", fieldMap); map.put("solrDeployPath", "http://127.0.0.1:8983/solr"); s.setProperties(map);
@Override public boolean equals(Object obj) { if (this == obj) { return true; } if (getClass() != obj.getClass()) { return false; } StageGroup g = (StageGroup) obj; if(g.getSize() == stages.size()) { for(Stage s : g.getStages()) { if(!hasStage(s.getName()) || !getStage(s.getName()).equals(s)) { return false; } } if((getPropertiesModifiedDate()==null) != (g.getPropertiesModifiedDate()==null)) { return false; } else if (getPropertiesModifiedDate()==null && g.getPropertiesModifiedDate() == null) { return true; } return getPropertiesModifiedDate().equals(g.getPropertiesModifiedDate()); } return false; }
if (stageGroup.getSize() == 1) { setParameters(stageGroup.getStages().iterator().next().getProperties());
if(s.getDatabaseFile().getId().equals(cmd.getOptionValue("i"))) { stagesToDelete.add(s); if(g.hasStage(s.getName())) { g.removeStage(s.getName());
public Stage getStage(String name) { for(Stage s : getStages()) { if(s.getName().equals(name)) { return s; } } return null; }
private void getPropetries(HttpRequest request, HttpResponse response, HttpContext context) { logger.trace("handleGetProperties()"); String stage = RESTTools.getStage(request); logger.debug("Received getProperties()-request for stage: "+stage); if(stage==null) { HttpResponseWriter.printMissingParameter(response, HttpEndpointConstants.STAGE_PARAM); return; } Map<String, Object> map = new HashMap<String, Object>(); if(reader.getPipeline().hasStage(stage)) { map = reader.getPipeline().getStage(stage).getProperties(); } else if(reader.getDebugPipeline().hasStage(stage)){ map = reader.getDebugPipeline().getStage(stage).getProperties(); } HttpResponseWriter.printJson(response, map); }
public void write(Stage stage, String group) throws IOException { DBObject q = reader.getStageQuery(stage.getName()); WriteResult result = stages.update(q, getStageDBObject(stage, group), true, false, concern); if (logger.isDebugEnabled()) { logger.debug("Wrote stage '{}' in group '{}', operation updated '{}' objects, got message '{}'", stage.getName(), group, result.getN(), result); } }
public void inactivate(Stage stage) { DBObject q = reader.getStageQuery(stage.getName()); stages.findAndModify(q, new BasicDBObject("$set", new BasicDBObject(MongoPipelineReader.ACTIVE_KEY, Stage.Mode.INACTIVE.toString()))); }
for (Stage stage : stages) { if (cmd.getStageNames() != null) { if (cmd.getStageNames().contains(stage.getName())) { log.info("Preparing to upload stage, " + stage.getName()); StageGroup g = new StageGroup(stage.getName()); g.addStage(stage); pipeline.addGroup(g); log.info("Preparing to upload stage, " + stage.getName()); StageGroup g = new StageGroup(stage.getName()); g.addStage(stage); pipeline.addGroup(g);