public List<Stage> createStages(PipelineConfiguration pipelineConfig) { List<Stage> stages = new ArrayList<Stage>(); for (Map<String, Object> stageConfig : pipelineConfig) { String jarId = (String) stageConfig.get(JAR_ID); String stageName = (String) stageConfig.get(STAGE_NAME); DatabaseFile df = new DatabaseFile(); df.setId(jarId); Stage s = new Stage(stageName, df); s.setProperties(stageConfig); s.setMode(Stage.Mode.ACTIVE); stages.add(s); } return stages; } }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; DatabaseFile other = (DatabaseFile) obj; if(equal(other.getFilename(), filename) && equal(other.getUploadDate(), getUploadDate()) && equal(other.getId(), id)) { return true; } return false; }
private DatabaseFile getFile(DBObject dbo) { if(dbo == null) { return null; } DatabaseFile df = new DatabaseFile(); df.setFilename((String)dbo.get("filename")); df.setUploadDate((Date)dbo.get("uploadDate")); df.setId(dbo.get(MongoDocument.MONGO_ID_KEY)); return df; } }
@Override public InputStream getStream(DatabaseFile df) { GridFSDBFile file = pipelinefs.findOne(new BasicDBObject(MongoDocument.MONGO_ID_KEY, df.getId())); return file==null ? null : file.getInputStream(); }
File f = new File(targetDirectory, df.getFilename()); files.add(f); InputStream dfis = df.getInputStream(); assert(dfis != null); FileOutputStream fos = new FileOutputStream(f);
private boolean attachFiles(StageGroup group) { Set<DatabaseFile> files = group.getDatabaseFiles(); if(files == null) { return false; } for(DatabaseFile file : group.getDatabaseFiles()) { file.attach(dbc.getPipelineReader().getStream(file)); } return true; }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (getClass() != obj.getClass()) return false; Stage s = (Stage) obj; if(!s.getName().equals(name) || !s.getPropertiesModifiedDate().equals(getPropertiesModifiedDate()) || !mode.equals(s.getMode())) { return false; } if(getDatabaseFile()==null) { return s.getDatabaseFile() == null; } return getDatabaseFile().equals(s.getDatabaseFile()); } }
/** * May return null if and only if a referenced stage library was not found in the database. */ public Set<DatabaseFile> getDatabaseFiles() { Map<Object, DatabaseFile> files = new HashMap<Object, DatabaseFile>(); for(Stage s : stages.values()) { if(s.getDatabaseFile()==null) { logger.error("Stage group '"+s.getName()+"' is missing it's library file"); return null; } else if(!files.containsKey(s.getDatabaseFile().getId())) { files.put(s.getDatabaseFile().getId(), s.getDatabaseFile()); } } return new HashSet<DatabaseFile>(files.values()); }
public static Stage getStage(Pipeline c, Object id, String stageName, String className, String ... afterStage) throws Exception { Stage s = new Stage(stageName, new DatabaseFile()); s.getDatabaseFile().setId(id); StageGroup g = new StageGroup(stageName); c.addGroup(g); g.addStage(s); Map<String, Object> props = new HashMap<String, Object>(); props.put("stageClass", "com.findwise.hydra."+className); if(afterStage!=null && afterStage.length>0) { props.put("queryOptions", new String[]{"touched("+afterStage[0]+",true)"}); } s.setProperties(props); return s; }
if(s.getDatabaseFile().getId().equals(cmd.getOptionValue("i"))) { stagesToDelete.add(s);
DatabaseFile df = new DatabaseFile(); try { df.setId(new ObjectId(libraryId)); } catch (Exception e) { df.setId(libraryId);
private DBObject getStageDBObject(Stage s, String group) { BasicDBObject obj = new BasicDBObject(); obj.put(MongoPipelineReader.STAGE_KEY, s.getName()); obj.put(MongoPipelineReader.TYPE_KEY, MongoPipelineReader.STAGE_TYPE); BasicDBObject props = new BasicDBObject(); if(s.isPropertiesChanged()) { props.put(MongoPipelineReader.PROPERTIES_DATE_SUBKEY, new Date()); } else { props.put(MongoPipelineReader.PROPERTIES_DATE_SUBKEY, s.getPropertiesModifiedDate()); } props.put(MongoPipelineReader.PROPERTIES_MAP_SUBKEY, s.getProperties()); obj.put(MongoPipelineReader.PROPERTIES_KEY, props); obj.put(MongoPipelineReader.ACTIVE_KEY, s.getMode().toString()); obj.put(MongoPipelineReader.GROUP_KEY, group); if(s.getDatabaseFile()!=null) { obj.put(MongoPipelineReader.FILE_KEY, s.getDatabaseFile().getId()); } return obj; }