public Object getFieldValue(_Fields field) { switch (field) { case INPUTS: return get_inputs(); case STREAMS: return get_streams(); case PARALLELISM_HINT: return Integer.valueOf(get_parallelism_hint()); case JSON_CONF: return get_json_conf(); } throw new IllegalStateException(); }
protected void initCommon(String id, IComponent component, Number parallelism) throws IllegalArgumentException { ComponentCommon common = new ComponentCommon(); common.set_inputs(new HashMap<GlobalStreamId, Grouping>()); if (parallelism != null) { int dop = parallelism.intValue(); if(dop < 1) { throw new IllegalArgumentException("Parallelism must be positive."); } common.set_parallelism_hint(dop); } else { common.set_parallelism_hint(1); } Map conf = component.getComponentConfiguration(); if(conf!=null) common.set_json_conf(JSONValue.toJSONString(conf)); _commons.put(id, common); }
/** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ public boolean isSet(_Fields field) { if (field == null) { throw new IllegalArgumentException(); } switch (field) { case INPUTS: return is_set_inputs(); case STREAMS: return is_set_streams(); case PARALLELISM_HINT: return is_set_parallelism_hint(); case JSON_CONF: return is_set_json_conf(); } throw new IllegalStateException(); }
private static ComponentCommon mkComponentcommon(Map<GlobalStreamId, Grouping> inputs, HashMap<String, StreamInfo> output_spec, Integer parallelism_hint) { ComponentCommon ret = new ComponentCommon(inputs, output_spec); if (parallelism_hint != null) { ret.set_parallelism_hint(parallelism_hint); } return ret; }
public static int parallelismHint(ComponentCommon component_common) { int phint = component_common.get_parallelism_hint(); if (!component_common.is_set_parallelism_hint()) { phint = 1; } return phint; }
case INPUTS: if (value == null) { unset_inputs(); } else { set_inputs((Map<GlobalStreamId,Grouping>)value); unset_streams(); } else { set_streams((Map<String,StreamInfo>)value); unset_parallelism_hint(); } else { set_parallelism_hint((Integer)value); unset_json_conf(); } else { set_json_conf((String)value);
/** * Add watermark stream to source components of window bolts */ private void maybeAddWatermarkInputs(ComponentCommon common, IRichBolt bolt) { if (bolt instanceof WindowedBoltExecutor) { Set<String> comps = new HashSet<>(); for (GlobalStreamId globalStreamId : common.get_inputs().keySet()) { comps.add(globalStreamId.get_componentId()); } for (String comp : comps) { common.put_to_inputs( new GlobalStreamId(comp, Common.WATERMARK_STREAM_ID), Grouping.all(new NullStruct())); } } }
protected BoltDeclarer grouping(String componentId, String streamId, Grouping grouping) { // Check if bolt is KvStateBolt, if so, enable the key range hash in upstream component TransactionBolt bolt = (TransactionBolt) _bolts.get(_boltId); if (bolt.getBoltExecutor() instanceof KvStatefulBoltExecutor) { ComponentCommon common = _commons.get(componentId); Map<String, Object> conf = new HashMap<>(); conf.put(ConfigExtension.ENABLE_KEY_RANGE_FIELD_GROUP, true); String currConf = common.get_json_conf(); common.set_json_conf(JStormUtils.mergeIntoJson(JStormUtils.parseJson(currConf), conf)); } // Add barrier snapshot stream for transaction topology Set<String> downstreamBolts = upToDownstreamComponentsMap.get(componentId); if (downstreamBolts != null && !downstreamBolts.contains(_boltId)) { downstreamBolts.add(_boltId); _commons.get(_boltId).put_to_inputs(new GlobalStreamId(componentId, TransactionCommon.BARRIER_STREAM_ID), Grouping.all(new NullStruct())); } _commons.get(_boltId).put_to_inputs(new GlobalStreamId(componentId, streamId), grouping); //return this; return super.grouping(componentId, streamId, grouping); } }
/** * Gets the declared inputs to the specified component. * * @return A map from subscribed component/stream to the grouping subscribed with. */ public Map<GlobalStreamId, Grouping> getSources(String componentId) { return getComponentCommon(componentId).get_inputs(); }
if (paraNum != null) { LOG.info("Set " + componentName + " as " + paraNum); common.set_parallelism_hint(paraNum); if (paraNum != null) { LOG.info("Set " + componentName + " as " + paraNum); common.set_parallelism_hint(paraNum); if (paraNum != null) { LOG.info("Set " + componentName + " as " + paraNum); common.set_parallelism_hint(paraNum); String jsonConfString = common.get_json_conf(); if (jsonConfString != null) { componentMap.putAll((Map) JStormUtils.from_json(jsonConfString)); common.set_parallelism_hint(taskNum); LOG.info("Set " + componentName + " parallelism " + taskNum); common.set_json_conf(JStormUtils.to_json(componentMap));
public void write(org.apache.thrift.protocol.TProtocol oprot, ComponentCommon struct) throws org.apache.thrift.TException { struct.validate(); if (struct.is_set_parallelism_hint()) { oprot.writeFieldBegin(PARALLELISM_HINT_FIELD_DESC); oprot.writeI32(struct.parallelism_hint); if (struct.is_set_json_conf()) { oprot.writeFieldBegin(JSON_CONF_FIELD_DESC); oprot.writeString(struct.json_conf);
public void validate() throws org.apache.thrift.TException { // check for required fields if (!is_set_inputs()) { throw new org.apache.thrift.protocol.TProtocolException("Required field 'inputs' is unset! Struct:" + toString()); } if (!is_set_streams()) { throw new org.apache.thrift.protocol.TProtocolException("Required field 'streams' is unset! Struct:" + toString()); } // check for sub-struct validity }
if (other.is_set_inputs()) { Map<GlobalStreamId,Grouping> __this__inputs = new HashMap<GlobalStreamId,Grouping>(other.inputs.size()); for (Map.Entry<GlobalStreamId, Grouping> other_element : other.inputs.entrySet()) { if (other.is_set_streams()) { Map<String,StreamInfo> __this__streams = new HashMap<String,StreamInfo>(other.streams.size()); for (Map.Entry<String, StreamInfo> other_element : other.streams.entrySet()) { if (other.is_set_json_conf()) { this.json_conf = other.json_conf;
/** * Gets the set of streams declared for the specified component. */ public Set<String> getComponentStreams(String componentId) { return getComponentCommon(componentId).get_streams().keySet(); }
@Override public void read(org.apache.thrift.protocol.TProtocol prot, StateSpoutSpec struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; struct.state_spout_object = new ComponentObject(); struct.state_spout_object.read(iprot); struct.set_state_spout_object_isSet(true); struct.common = new ComponentCommon(); struct.common.read(iprot); struct.set_common_isSet(true); } }
public static Integer componentParalism(Map stormConf, ComponentCommon common) { Map mergeMap = new HashMap(); mergeMap.putAll(stormConf); String jsonConfString = common.get_json_conf(); if (jsonConfString != null) { Map componentMap = (Map) JStormUtils.from_json(jsonConfString); mergeMap.putAll(componentMap); } Integer taskNum = common.get_parallelism_hint(); // don't get taskNum from component configuraiton // skip .setTaskNum // Integer taskNum = null; // Object taskNumObject = mergeMap.get(Config.TOPOLOGY_TASKS); // if (taskNumObject != null) { // taskNum = JStormUtils.parseInt(taskNumObject); // } else { // taskNum = common.get_parallelism_hint(); // if (taskNum == null) { // taskNum = Integer.valueOf(1); // } // } Object maxTaskParalismObject = mergeMap.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM); if (maxTaskParalismObject == null) { return taskNum; } else { int maxTaskParalism = JStormUtils.parseInt(maxTaskParalismObject); return Math.min(maxTaskParalism, taskNum); } }
/** * get component configuration */ @SuppressWarnings("unchecked") public static Map component_conf(TopologyContext topology_context, String component_id) { Map<Object, Object> componentConf = new HashMap<>(); String jconf = topology_context.getComponentCommon(component_id).get_json_conf(); if (jconf != null) { componentConf = (Map<Object, Object>) JStormUtils.from_json(jconf); } return componentConf; }
@Override public boolean equals(Object that) { if (that == null) return false; if (that instanceof ComponentCommon) return this.equals((ComponentCommon)that); return false; }
common.put_to_streams(ACKER_ACK_STREAM_ID, Thrift.outputFields(ackList)); common.put_to_streams(ACKER_FAIL_STREAM_ID, Thrift.outputFields(failList)); ComponentCommon common = bolt.get_common(); List<String> initList = JStormUtils.mk_list("id", "init-val", "spout-task"); common.put_to_streams(ACKER_INIT_STREAM_ID, Thrift.outputFields(initList)); common.put_to_inputs(ack_ack, Thrift.mkDirectGrouping()); common.put_to_inputs(ack_fail, Thrift.mkDirectGrouping());
@Override public T addConfigurations(Map conf) { if(conf!=null && conf.containsKey(Config.TOPOLOGY_KRYO_REGISTER)) { throw new IllegalArgumentException("Cannot set serializations for a component using fluent API"); } String currConf = _commons.get(_id).get_json_conf(); _commons.get(_id).set_json_conf(mergeIntoJson(parseJson(currConf), conf)); return (T) this; } }