public TupleImpl(GeneralTopologyContext context, List<Object> values, String srcComponent, int taskId, String streamId, MessageId id) { this.values = context.doSanityCheck() ? Collections.unmodifiableList(values) : values; this.taskId = taskId; this.streamId = streamId; this.id = id; this.context = context; this.srcComponent = srcComponent; if (context.doSanityCheck()) { String componentId = context.getComponentId(taskId); Fields schema = context.getComponentOutputFields(componentId, streamId); if (values.size() != schema.size()) { throw new IllegalArgumentException("Tuple created with wrong number of fields. Expected " + schema.size() + " fields but got " + values.size() + " fields"); } } }
public int maxTopologyMessageTimeout() { Integer max = ObjectReader.getInt(_topoConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); for (String spout : getRawTopology().get_spouts().keySet()) { ComponentCommon common = getComponentCommon(spout); String jsonConf = common.get_json_conf(); if (jsonConf != null) { try { Map<String, Object> conf = (Map) JSONValue.parseWithException(jsonConf); Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); max = Math.max(ObjectReader.getInt(comp, max), max); } catch (ParseException e) { throw new RuntimeException(e); } } } return max; }
/** * Gets information about who is consuming the outputs of the specified component, and how. * * @return Map from stream id to component id to the Grouping used. */ public Map<String, Map<String, Grouping>> getTargets(String componentId) { Map<String, Map<String, Grouping>> ret = new HashMap<>(); for (String otherComponentId : getComponentIds()) { Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs(); for (Map.Entry<GlobalStreamId, Grouping> entry : inputs.entrySet()) { GlobalStreamId id = entry.getKey(); if (id.get_componentId().equals(componentId)) { Map<String, Grouping> curr = ret.get(id.get_streamId()); if (curr == null) { curr = new HashMap<>(); } curr.put(otherComponentId, entry.getValue()); ret.put(id.get_streamId(), curr); } } } return ret; }
@Override public TupleImpl deserialize(byte[] ser) { try { _kryoInput.setBuffer(ser); int taskId = _kryoInput.readInt(true); int streamId = _kryoInput.readInt(true); String componentName = _context.getComponentId(taskId); String streamName = _ids.getStreamName(componentName, streamId); MessageId id = MessageId.deserialize(_kryoInput); List<Object> values = _kryo.deserializeFrom(_kryoInput); return new TupleImpl(_context, values, componentName, taskId, streamName, id); } catch (IOException e) { throw new RuntimeException(e); } } }
/** * Gets a list of all component ids in this topology */ public Set<String> getComponentIds() { return ThriftTopologyUtils.getComponentIds(getRawTopology()); }
public TupleImpl(GeneralTopologyContext context, List<Object> values, int taskId, String streamId, MessageId id) { this.values = values; this.taskId = taskId; this.streamId = streamId; this.id = id; this.context = context; String componentId = context.getComponentId(taskId); Fields schema = context.getComponentOutputFields(componentId, streamId); if(values.size()!=schema.size()) { throw new IllegalArgumentException( "Tuple created with wrong number of fields. " + "Expected " + schema.size() + " fields but got " + values.size() + " fields"); } }
/** * Gets the set of streams declared for the specified component. */ public Set<String> getComponentStreams(String componentId) { return getComponentCommon(componentId).get_streams().keySet(); }
/** * Gets the declared output fields for the specified global stream id. */ public Fields getComponentOutputFields(GlobalStreamId id) { return getComponentOutputFields(id.get_componentId(), id.get_streamId()); }
private Tuple generateTestTuple(Long key, String value) { TopologyBuilder builder = new TopologyBuilder(); GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") { @Override public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("key", "value"); } }; return new TupleImpl(topologyContext, new Values(key, value), topologyContext.getComponentId(1), 1, ""); }
public ComponentCommon getComponentCommon(String componentId) { return ThriftTopologyUtils.getComponentCommon(getRawTopology(), componentId); }
/** * Gets the declared inputs to the specified component. * * @return A map from subscribed component/stream to the grouping subscribed with. */ public Map<GlobalStreamId, Grouping> getSources(String componentId) { return getComponentCommon(componentId).get_inputs(); }
@Override public Fields getFields() { return context.getComponentOutputFields(getSourceComponent(), getSourceStreamId()); }
private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) { TopologyBuilder builder = new TopologyBuilder(); GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") { @Override public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("id", "msg", "city", "state"); } }; return new TupleImpl(topologyContext, new Values(id, msg, city, state), topologyContext.getComponentId(1), 1, ""); }
public KryoTupleDeserializer(final Map<String, Object> conf, final GeneralTopologyContext context) { _kryo = new KryoValuesDeserializer(conf); _context = context; _ids = new SerializationFactory.IdDictionary(context.getRawTopology()); _kryoInput = new Input(1); }
public int maxTopologyMessageTimeout() { Integer max = Utils.getInt(_stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)); for(String spout: getRawTopology().get_spouts().keySet()) { ComponentCommon common = getComponentCommon(spout); String jsonConf = common.get_json_conf(); if(jsonConf!=null) { try { Map conf = (Map) JSONValue.parseWithException(jsonConf); Object comp = conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS); max = Math.max(Utils.getInt(comp, max), max); } catch (ParseException e) { throw new RuntimeException(e); } } } return max; } }
/** * Gets information about who is consuming the outputs of the specified component, * and how. * * @return Map from stream id to component id to the Grouping used. */ public Map<String, Map<String, Grouping>> getTargets(String componentId) { Map<String, Map<String, Grouping>> ret = new HashMap<>(); for(String otherComponentId: getComponentIds()) { Map<GlobalStreamId, Grouping> inputs = getComponentCommon(otherComponentId).get_inputs(); for(Map.Entry<GlobalStreamId, Grouping> entry: inputs.entrySet()) { GlobalStreamId id = entry.getKey(); if(id.get_componentId().equals(componentId)) { Map<String, Grouping> curr = ret.get(id.get_streamId()); if(curr==null) curr = new HashMap<>(); curr.put(otherComponentId, entry.getValue()); ret.put(id.get_streamId(), curr); } } } return ret; }
/** * Gets the set of streams declared for the specified component. */ public Set<String> getComponentStreams(String componentId) { return getComponentCommon(componentId).get_streams().keySet(); }
/** * Gets the declared output fields for the specified global stream id. */ public Fields getComponentOutputFields(GlobalStreamId id) { return getComponentOutputFields(id.get_componentId(), id.get_streamId()); }
private static Tuple generateTestTuple(GenericRecord record) { TopologyBuilder builder = new TopologyBuilder(); GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap(), new HashMap(), new HashMap(), "") { @Override public Fields getComponentOutputFields(String componentId, String streamId) { return new Fields("record"); } }; return new TupleImpl(topologyContext, new Values(record), topologyContext.getComponentId(1), 1, ""); }
public KryoTupleSerializer(final Map<String, Object> conf, final GeneralTopologyContext context) { _kryo = new KryoValuesSerializer(conf); _kryoOut = new Output(2000, 2000000000); _ids = new SerializationFactory.IdDictionary(context.getRawTopology()); }