public static Map<String, Object> toHierarchicalMap(final Props props) { final Map<String, Object> propsMap = new HashMap<>(); propsMap.put("source", props.getSource()); propsMap.put("props", toStringMap(props, true)); if (props.getParent() != null) { propsMap.put("parent", toHierarchicalMap(props.getParent())); } return propsMap; }
public static Props fromHierarchicalMap(final Map<String, Object> propsMap) { if (propsMap == null) { return null; } final String source = (String) propsMap.get("source"); final Map<String, String> propsParams = (Map<String, String>) propsMap.get("props"); final Map<String, Object> parent = (Map<String, Object>) propsMap.get("parent"); final Props parentProps = fromHierarchicalMap(parent); final Props props = new Props(parentProps, propsParams); props.setSource(source); return props; }
@Override public List<Pair<String, Props>> handle(final ResultSet rs) throws SQLException { if (!rs.next()) { return Collections.emptyList(); } final List<Pair<String, Props>> properties = new ArrayList<>(); do { final String name = rs.getString(3); final int eventType = rs.getInt(5); final byte[] dataBytes = rs.getBytes(6); final EncodingType encType = EncodingType.fromInteger(eventType); String propertyString = null; try { if (encType == EncodingType.GZIP) { // Decompress the sucker. propertyString = GZIPUtils.unGzipString(dataBytes, "UTF-8"); } else { propertyString = new String(dataBytes, "UTF-8"); } final Props props = PropsUtils.fromJSONString(propertyString); props.setSource(name); properties.add(new Pair<>(name, props)); } catch (final IOException e) { throw new SQLException(e); } } while (rs.next()); return properties; } }
/** * Load properties from the given path * * @param jobPath The path to load from * @param props The parent properties for loaded properties * @param suffixes The suffixes of files to load */ public static void loadPropsBySuffix(final File jobPath, final Props props, final String... suffixes) { try { if (jobPath.isDirectory()) { final File[] files = jobPath.listFiles(); if (files != null) { for (final File file : files) { loadPropsBySuffix(file, props, suffixes); } } } else if (endsWith(jobPath, suffixes)) { props.putAll(new Props(null, jobPath.getAbsolutePath())); } } catch (final IOException e) { throw new RuntimeException("Error loading schedule properties.", e); } }
public static String toJSONString(final Props props, final boolean localOnly) { final Map<String, String> map = toStringMap(props, localOnly); return JSONUtils.toJSON(map); }
protected void resolveProps() { this.jobProps = PropsUtils.resolveProps(this.jobProps); }
public void uploadExecutableNode(final ExecutableNode node, final Props inputProps) throws ExecutorManagerException { final String INSERT_EXECUTION_NODE = "INSERT INTO execution_jobs " + "(exec_id, project_id, version, flow_id, job_id, start_time, " + "end_time, status, input_params, attempt) VALUES (?,?,?,?,?,?,?,?,?,?)"; byte[] inputParam = null; if (inputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(inputProps)); inputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } final ExecutableFlow flow = node.getExecutableFlow(); final String flowId = node.getParentFlow().getFlowPath(); logger.info("Uploading flowId " + flowId); try { this.dbOperator.update(INSERT_EXECUTION_NODE, flow.getExecutionId(), flow.getProjectId(), flow.getVersion(), flowId, node.getId(), node.getStartTime(), node.getEndTime(), node.getStatus().getNumVal(), inputParam, node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error writing job " + node.getId(), e); } }
/** * Load job schedules from the given directories * * @param dir The directory to look in * @param suffixes File suffixes to load * @return The loaded set of schedules */ public static Props loadPropsInDir(final File dir, final String... suffixes) { return loadPropsInDir(null, dir, suffixes); }
@Test public void testGetPropertyDiff() throws IOException { final Props oldProps = new Props(); final Props newProps1 = new Props(); oldProps.put("a", "a_value1"); oldProps.put("b", "b_value1"); newProps1.put("b", "b_value2"); final String message1 = PropsUtils.getPropertyDiff(oldProps, newProps1); Assert.assertEquals(message1, "Deleted Properties: [ a, a_value1], \nModified Properties: [ b, b_value1-->b_value2], "); final Props newProps2 = new Props(); newProps2.put("a", "a_value1"); newProps2.put("b", "b_value1"); newProps2.put("c", "c_value1"); final String message2 = PropsUtils.getPropertyDiff(oldProps, newProps2); Assert.assertEquals(message2, "Newly created Properties: [ c, c_value1], \n"); final Props newProps3 = new Props(); newProps3.put("b", "b_value1"); newProps3.put("c", "a_value1"); final String message3 = PropsUtils.getPropertyDiff(oldProps, newProps3); Assert.assertEquals(message3, "Newly created Properties: [ c, a_value1], \nDeleted Properties: [ a, a_value1], \n"); }
/** * Load job schedules from the given directories * * @param parent The parent properties for these properties * @param dir The directory to look in * @param suffixes File suffixes to load * @return The loaded set of schedules */ public static Props loadPropsInDir(final Props parent, final File dir, final String... suffixes) { try { final Props props = new Props(parent); final File[] files = dir.listFiles(); Arrays.sort(files); if (files != null) { for (final File f : files) { if (f.isFile() && endsWith(f, suffixes)) { props.putAll(new Props(null, f.getAbsolutePath())); } } } return props; } catch (final IOException e) { throw new RuntimeException("Error loading properties.", e); } }
objMap.put(OUTPUT_PROPS_PARAM, PropsUtils.toStringMap(this.outputProps, true));
private void failIfNotException(final Props props) { try { PropsUtils.resolveProps(props); Assert.fail(); } catch (final UndefinedPropertyException e) { e.printStackTrace(); } catch (final IllegalArgumentException e) { e.printStackTrace(); } } }
public void updateExecutableNode(final ExecutableNode node) throws ExecutorManagerException { final String UPSERT_EXECUTION_NODE = "UPDATE execution_jobs " + "SET start_time=?, end_time=?, status=?, output_params=? " + "WHERE exec_id=? AND flow_id=? AND job_id=? AND attempt=?"; byte[] outputParam = null; final Props outputProps = node.getOutputProps(); if (outputProps != null) { try { final String jsonString = JSONUtils.toJSON(PropsUtils.toHierarchicalMap(outputProps)); outputParam = GZIPUtils.gzipString(jsonString, "UTF-8"); } catch (final IOException e) { throw new ExecutorManagerException("Error encoding input params"); } } try { this.dbOperator.update(UPSERT_EXECUTION_NODE, node.getStartTime(), node .getEndTime(), node.getStatus().getNumVal(), outputParam, node .getExecutableFlow().getExecutionId(), node.getParentFlow() .getFlowPath(), node.getId(), node.getAttempt()); } catch (final SQLException e) { throw new ExecutorManagerException("Error updating job " + node.getId(), e); } }
/** * Load properties from the given path * * @param jobPath The path to load from * @param props The parent properties for loaded properties * @param suffixes The suffixes of files to load */ public static void loadPropsBySuffix(final File jobPath, final Props props, final String... suffixes) { try { if (jobPath.isDirectory()) { final File[] files = jobPath.listFiles(); if (files != null) { for (final File file : files) { loadPropsBySuffix(file, props, suffixes); } } } else if (endsWith(jobPath, suffixes)) { props.putAll(new Props(null, jobPath.getAbsolutePath())); } } catch (final IOException e) { throw new RuntimeException("Error loading schedule properties.", e); } }
/** * Load job schedules from the given directories * * @param dirs The directories to check for properties * @param suffixes The suffixes to load * @return The properties */ public static Props loadPropsInDirs(final List<File> dirs, final String... suffixes) { final Props props = new Props(); for (final File dir : dirs) { props.putLocal(loadPropsInDir(dir, suffixes)); } return props; }
final String diffMessage = PropsUtils.getPropertyDiff(oldProps, prop);
/** * Load job schedules from the given directories * * @param parent The parent properties for these properties * @param dir The directory to look in * @param suffixes File suffixes to load * @return The loaded set of schedules */ public static Props loadPropsInDir(final Props parent, final File dir, final String... suffixes) { try { final Props props = new Props(parent); final File[] files = dir.listFiles(); Arrays.sort(files); if (files != null) { for (final File f : files) { if (f.isFile() && endsWith(f, suffixes)) { props.putAll(new Props(null, f.getAbsolutePath())); } } } return props; } catch (final IOException e) { throw new RuntimeException("Error loading properties.", e); } }
public static Map<String,Object> toHierarchicalMap(Props props) { Map<String,Object> propsMap = new HashMap<String,Object>(); propsMap.put("source", props.getSource()); propsMap.put("props", toStringMap(props, true)); if (props.getParent() != null) { propsMap.put("parent", toHierarchicalMap(props.getParent())); } return propsMap; } }