public DelimitedRecordHiveMapper withColumnFields(Fields columnFields) { this.columnFields = columnFields; List<String> tempColumnNamesList = this.columnFields.toList(); columnNames = new String[tempColumnNamesList.size()]; tempColumnNamesList.toArray(columnNames); return this; }
public SimpleTuple(Fields keyFields, List<Object> values) { this.keys = keyFields.toList(); this.values = new ArrayList<>(); this.values.addAll(values); while (this.values.size() < keys.size()) { this.values.add(null); } }
public SimpleTuple(Fields keyFields, List<Object>... values) { this.keys = keyFields.toList(); this.values = new ArrayList<>(); for (List<Object> valueList : values) { this.values.addAll(valueList); } while (this.values.size() < keys.size()) { this.values.add(null); } }
public SimpleJdbcLookupMapper(Fields outputFields, List<Column> queryColumns) { super(queryColumns); Validate.notEmpty(outputFields.toList()); this.outputFields = outputFields; }
public static Fields fieldsUnion(Fields... fields) { Set<String> ret = new HashSet<String>(); for (Fields f : fields) { if (f != null) { ret.addAll(f.toList()); } } return new Fields(new ArrayList<String>(ret)); }
public void declareStream(String streamId, boolean direct, Fields fields) { if (null == streamId) { throw new IllegalArgumentException("streamId can't be null"); } if (_fields.containsKey(streamId)) { throw new IllegalArgumentException("Fields for " + streamId + " already set"); } _fields.put(streamId, new StreamInfo(fields.toList(), direct)); }
public static Fields fieldsConcat(Fields... fields) { List<String> ret = new ArrayList<String>(); for (Fields f : fields) { if (f != null) { ret.addAll(f.toList()); } } return new Fields(ret); }
/** * Gets the declared output fields for all streams for the component this task is a part of. */ public Map<String, List<String>> getThisOutputFieldsForStreams() { Map<String, List<String>> streamToFields = new HashMap<>(); for (String stream : this.getThisStreams()) { streamToFields.put(stream, this.getThisOutputFields(stream).toList()); } return streamToFields; }
@Override public void declareOutputFields(OutputFieldsDeclarer declarer) { List<String> fields = new ArrayList<>(_spout.getOutputFields().toList()); fields.add(0, ID_FIELD); declarer.declareStream(_streamName, new Fields(fields)); }
public BoltDeclarer fieldsGrouping(String componentId, String streamId, Fields fields) { return grouping(componentId, streamId, Grouping.fields(fields.toList())); }
/** * Gets the declared input fields for this component. * * @return A map from sources to streams to fields. */ public Map<String, Map<String, List<String>>> getThisInputFields() { Map<String, Map<String, List<String>>> outputMap = new HashMap<>(); for (Map.Entry<GlobalStreamId, Grouping> entry : this.getThisSources().entrySet()) { String componentId = entry.getKey().get_componentId(); Set<String> streams = getComponentStreams(componentId); for (String stream : streams) { Map<String, List<String>> streamFieldMap = outputMap.get(componentId); if (streamFieldMap == null) { streamFieldMap = new HashMap<>(); outputMap.put(componentId, streamFieldMap); } streamFieldMap.put(stream, getComponentOutputFields(componentId, stream).toList()); } } return outputMap; }
public InitialReceiver(String stream, Fields allFields) { _stream = stream; _factory = new RootFactory(allFields); List<String> projected = new ArrayList<>(allFields.toList()); projected.remove(0); _project = new ProjectionFactory(_factory, new Fields(projected)); }
/** * ## Repartitioning Operation * * @param fields * @return */ public Stream partitionBy(Fields fields) { projectionValidation(fields); return partition(Grouping.fields(fields.toList())); }
@Override public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) { _fieldLocations = new HashMap<String, GlobalStreamId>(); _collector = collector; int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue(); _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback()); _numSources = context.getThisSources().size(); Set<String> idFields = null; for (GlobalStreamId source : context.getThisSources().keySet()) { Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId()); Set<String> setFields = new HashSet<String>(fields.toList()); if (idFields == null) { idFields = setFields; } else { idFields.retainAll(setFields); } for (String outfield : _outFields) { for (String sourcefield : fields) { if (outfield.equals(sourcefield)) { _fieldLocations.put(outfield, source); } } } } _idFields = new Fields(new ArrayList<String>(idFields)); if (_fieldLocations.size() != _outFields.size()) { throw new RuntimeException("Cannot find all outfields among sources"); } }
void declareOutputFields(OutputFieldsDeclarer declarer) { for (ProcessorNode node : nodes) { for (String stream : node.getOutputStreams()) { if (timestampField == null) { declarer.declareStream(stream, node.getOutputFields()); } else { List<String> fields = new ArrayList<>(); fields.addAll(node.getOutputFields().toList()); fields.add(timestampField); declarer.declareStream(stream, new Fields(fields)); } /* * Declare a separate 'punctuation' stream per output stream so that the receiving bolt * can subscribe to this stream with 'ALL' grouping and process the punctuation once it * receives from all upstream tasks. */ declarer.declareStream(StreamUtil.getPunctuationStream(stream), StreamUtil.getPunctuationFields()); } } }
@Test public void toListTest() { Fields fields = new Fields("foo", "bar"); List<String> fieldList = fields.toList(); Assert.assertEquals(fieldList.size(), 2); Assert.assertEquals(fieldList.get(0), "foo"); Assert.assertEquals(fieldList.get(1), "bar"); }
@Test public void testTestTuple() throws Exception { Tuple tuple = Testing.testTuple(new Values("james", "bond")); assertThat(tuple.getValues(), is(new Values("james", "bond"))); assertThat(tuple.getSourceStreamId(), is(Utils.DEFAULT_STREAM_ID)); assertThat(tuple.getFields().toList(), is(Arrays.asList("field1", "field2"))); assertThat(tuple.getSourceComponent(), is("component")); }
@Test public void testTestTupleWithMkTupleParam() throws Exception { MkTupleParam mkTupleParam = new MkTupleParam(); mkTupleParam.setStream("test-stream"); mkTupleParam.setComponent("test-component"); mkTupleParam.setFields("fname", "lname"); Tuple tuple = Testing.testTuple(new Values("james", "bond"), mkTupleParam); assertThat(tuple.getValues(), is(new Values("james", "bond"))); assertThat(tuple.getSourceStreamId(), is("test-stream")); assertThat(tuple.getFields().toList(), is(Arrays.asList("fname", "lname"))); assertThat(tuple.getSourceComponent(), is("test-component")); }