private void checkAttributeValueSerializable(AttributeMap attributes, String context) { StringBuilder sb = new StringBuilder(); String delim = ""; // Check all attributes got operator are serializable for (Entry<Attribute<?>, Object> entry : attributes.entrySet()) { if (entry.getValue() != null && !(entry.getValue() instanceof Serializable)) { sb.append(delim).append(entry.getKey().getSimpleName()); delim = ", "; } } if (sb.length() > 0) { throw new ValidationException("Attribute value(s) for " + sb.toString() + " in " + context + " are not serializable"); } }
@GET @Path(PATH_LOGICAL_PLAN + "/attributes") @Produces(MediaType.APPLICATION_JSON) public JSONObject getApplicationAttributes(@QueryParam("attributeName") String attributeName) { init(); HashMap<String, String> map = new HashMap<>(); for (Map.Entry<Attribute<?>, Object> entry : dagManager.getApplicationAttributes().entrySet()) { if (attributeName == null || entry.getKey().getSimpleName().equals(attributeName)) { Map.Entry<Attribute<Object>, Object> entry1 = (Map.Entry<Attribute<Object>, Object>)(Map.Entry)entry; map.put(entry1.getKey().getSimpleName(), entry1.getKey().codec.toString(entry1.getValue())); } } return new JSONObject(map); }
/** * Copy attribute from source attributeMap to destination attributeMap. * * @param dest destination attribute map. * @param source source attribute map. */ private void copyAttributes(AttributeMap dest, AttributeMap source) { for (Entry<Attribute<?>, ?> a : source.entrySet()) { dest.put((Attribute<Object>)a.getKey(), a.getValue()); } }
@Override public YarnAppHandleImpl launchApp(final StreamingApplication app, Configuration conf, Attribute.AttributeMap launchParameters) throws LauncherException { if (launchParameters != null) { for (Map.Entry<Attribute<?>, Object> entry : launchParameters.entrySet()) { String property = propMapping.get(entry.getKey()); if (property != null) { setConfiguration(conf, property, entry.getValue()); } } } try { String name = app.getClass().getName(); StramAppLauncher appLauncher = new StramAppLauncher(name, conf); appLauncher.loadDependencies(); StreamingAppFactory appFactory = new StreamingAppFactory(name, app.getClass()) { @Override public LogicalPlan createApp(LogicalPlanConfiguration planConfig) { return super.createApp(app, planConfig); } }; ApplicationId appId = appLauncher.launchApp(appFactory); appLauncher.resetContextClassLoader(); return new YarnAppHandleImpl(appId, conf); } catch (Exception ex) { throw new LauncherException(ex); } }
/** * This method is for saving meta information about this application in HDFS -- the meta information that generally * does not change across multiple attempts */ private void saveMetaInfo() throws IOException { Path file = new Path(this.vars.appPath, APP_META_FILENAME + "." + System.nanoTime()); try (FSDataOutputStream os = fileContext.create(file, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), Options.CreateOpts.CreateParent.createParent())) { JSONObject top = new JSONObject(); JSONObject attributes = new JSONObject(); for (Map.Entry<Attribute<?>, Object> entry : this.plan.getLogicalPlan().getAttributes().entrySet()) { attributes.put(entry.getKey().getSimpleName(), entry.getValue()); } JSONObject autoMetrics = new JSONObject(); for (Map.Entry<String, Map<String, Object>> entry : latestLogicalMetrics.entrySet()) { autoMetrics.put(entry.getKey(), new JSONArray(entry.getValue().keySet())); } top.put(APP_META_KEY_ATTRIBUTES, attributes); top.put(APP_META_KEY_METRICS, autoMetrics); os.write(top.toString().getBytes()); } catch (JSONException ex) { throw new RuntimeException(ex); } Path origPath = new Path(this.vars.appPath, APP_META_FILENAME); fileContext.rename(file, origPath, Options.Rename.OVERWRITE); }
@GET @Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/attributes") @Produces(MediaType.APPLICATION_JSON) public JSONObject getOperatorAttributes(@PathParam("operatorName") String operatorName, @QueryParam("attributeName") String attributeName) { init(); OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName); if (logicalOperator == null) { throw new NotFoundException(); } HashMap<String, String> map = new HashMap<>(); for (Map.Entry<Attribute<?>, Object> entry : dagManager.getOperatorAttributes(operatorName).entrySet()) { if (attributeName == null || entry.getKey().getSimpleName().equals(attributeName)) { Map.Entry<Attribute<Object>, Object> entry1 = (Map.Entry<Attribute<Object>, Object>)(Map.Entry)entry; map.put(entry1.getKey().getSimpleName(), entry1.getKey().codec.toString(entry1.getValue())); } } return new JSONObject(map); }
@GET @Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/ports/{portName}/attributes") @Produces(MediaType.APPLICATION_JSON) public JSONObject getPortAttributes(@PathParam("operatorName") String operatorName, @PathParam("portName") String portName, @QueryParam("attributeName") String attributeName) { init(); OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName); if (logicalOperator == null) { throw new NotFoundException(); } HashMap<String, String> map = new HashMap<>(); for (Map.Entry<Attribute<?>, Object> entry : dagManager.getPortAttributes(operatorName, portName).entrySet()) { if (attributeName == null || entry.getKey().getSimpleName().equals(attributeName)) { Map.Entry<Attribute<Object>, Object> entry1 = (Map.Entry<Attribute<Object>, Object>)(Map.Entry)entry; map.put(entry1.getKey().getSimpleName(), entry1.getKey().codec.toString(entry1.getValue())); } } return new JSONObject(map); }