@Override public void process(Row input, Collector<Row> collector) { collector.collect(input); }
public static Row of(Object[] values) { return new DefaultRow(values); }
default <T> T getField(int i) { return getAs(i); }
public KafkaSink09(SinkContext context, Kafka09SinkConfig config) { schema = context.getSchema(); if (!Strings.isNullOrEmpty(config.idField)) { int fieldIndex = schema.getFieldIndex(config.idField); checkState(fieldIndex != -1, config.idField + " does not exist, only " + schema.getFields()); this.idIndex = fieldIndex; } this.config = config; this.topic = config.topics; }
@Override public void process(Row value) { Gson gson = new Gson(); Map<String, Object> map = new HashMap<>(); for (String fieldName : schema.getFieldNames()) { map.put(fieldName, value.getAs(fieldName)); } String message = gson.toJson(map); kafkaProducer.send(message); }
public HbaseSink(SinkContext context, HbaseConfig config) throws Exception { { this.config = config; schema = context.getSchema(); tableName = context.getSinkTable(); if (config.nameSpace != null) { tableName = config.nameSpace + ":" + tableName; } hbaseHelper = new HbaseHelper(tableName, config.zookeeper, config.zkNodeParent); if (!hbaseHelper.tableExist(tableName)) { throw new TableNotFoundException("table does not exist, table name " + tableName); } columMapping = ColumUtil.mapping(schema, config.columnMapping); if (!Strings.isNullOrEmpty(config.rowkey)) { int fieldIndex = schema.getFieldIndex(config.rowkey); checkState(fieldIndex != -1, config.rowkey + " does not exist, only " + schema.getFields()); this.rowkeyIndex = fieldIndex; } checkState(rowkeyIndex != -1, "`rowkey` must be set"); hbaseHelper.closeConnection(); } }
public HdfsSink(HdfsSinkConfig config, SinkContext context) { this.config = config; this.sinkTable = context.getSinkTable(); this.schema = context.getSchema(); checkState(sinkTable.length() > 0, "sinkTable is " + sinkTable); for (int i = 0; i < schema.getFieldNames().size(); i++) { if (schema.getFieldNames().get(i).equalsIgnoreCase(config.eventTimeName)) { this.eventTimeIndex = i; break; } } checkState(eventTimeIndex != -1, "eventTime_field " + config.eventTimeName + " does not exist,but only " + schema.getFieldNames()); checkState("text".equals(config.format.toLowerCase()) || "parquet".equals(config.format.toLowerCase()), "Hdfs sink format only supports text and parquet"); }
public ClickHouseSink(SinkContext context, ClickHouseSinkConfig clickHouseSinkConfig) { this.config = clickHouseSinkConfig; checkState(config.getQuery() != null, "insert into query not setting"); this.prepareStatementQuery = config.getQuery().replaceAll("\\$\\{.*?}", "?"); schema = context.getSchema(); Map<String, String> nt = new HashMap<String, String>(); for (int i = 0; i < schema.getFieldNames().size(); i++) { nt.put(schema.getFieldNames().get(i), schema.getFieldTypes().get(i).toString().split(" ")[1]); } this.nametypes = nt; }
public JsonSchema(SourceContext context) { ideal.sylph.etl.Row.Schema schema = context.getSchema(); TypeInformation<?>[] types = schema.getFieldTypes().stream().map(TypeExtractor::createTypeInfo).toArray(TypeInformation<?>[]::new); String[] names = schema.getFieldNames().toArray(new String[0]); this.rowTypeInfo = new RowTypeInfo(types, names); }
default String mkString() { return this.mkString(","); }
public static SchemaBuilder newBuilder() { return new SchemaBuilder(); }
public SchemaBuilder add(String name, Class<?> javaType) { fields.add(new Field(name, javaType)); return this; }
public Schema build() { return new Schema(fields.stream().collect(Collectors.toList())); } }
public Elasticsearch6Sink(SinkContext context, ElasticsearchSinkConfig config) { schema = context.getSchema(); this.config = config; if (!Strings.isNullOrEmpty(config.idField)) { int fieldIndex = schema.getFieldIndex(config.idField); checkState(fieldIndex != -1, config.idField + " does not exist, only " + schema.getFields()); this.idIndex = fieldIndex; } if (config.update) { checkState(idIndex != -1, "This is Update mode, `id_field` must be set"); } }
@Override public void process(Row value) { Object rowkey = value.getAs(rowkeyIndex); if (rowkey == null) { return; } Put put = new Put(BytesUtil.toBytes(rowkey)); try { for (String fieldName : schema.getFieldNames()) { if (!config.rowkey.equals(fieldName)) { Tuple2<String, String> tuple2 = columMapping.get(fieldName); if (tuple2 != null) { hbaseHelper.addColumn(tuple2.f0(), tuple2.f1(), value.getAs(fieldName), put); } else { logger.warn("Field:" + fieldName + " not defined in table " + tableName); } } } if (!put.isEmpty()) { hbaseHelper.store(put); } } catch (Exception e) { logger.error("put record to hbase fail.", e); } }
@Override public void process(Row value) { try { long eventTime = value.getAs(eventTimeIndex); hdfsFactory.writeLine(eventTime, value); } catch (ClassCastException e) { logger.error("eventTimeField {}, index [{}], but value is {}", config.eventTimeName, eventTimeIndex, value.getAs(eventTimeIndex)); try { TimeUnit.MILLISECONDS.sleep(1); } catch (InterruptedException e1) { Thread.currentThread().interrupt(); } } catch (IOException e) { logger.error("", e); } }
@Override public void process(Row value) { System.out.println(value.mkString()); } }
public Elasticsearch5Sink(SinkContext context, ElasticsearchSinkConfig config) { schema = context.getSchema(); this.config = config; if (!Strings.isNullOrEmpty(config.idField)) { int fieldIndex = schema.getFieldIndex(config.idField); checkState(fieldIndex != -1, config.idField + " does not exist, only " + schema.getFields()); this.idIndex = fieldIndex; } if (config.update) { checkState(idIndex != -1, "This is Update mode, `id_field` must be set"); } }
@Override public void process(Row row) { try { int i = 1; for (String key : keys) { Object value = isNumeric(key) ? row.getAs(Integer.parseInt(key)) : row.getAs(key); statement.setObject(i, value); i += 1; } statement.addBatch(); // submit batch if (num++ >= 50) { statement.executeBatch(); num = 0; } } catch (SQLException e) { throwsException(e); } }
@Override public void writeLine(long eventTime, Row row) throws IOException { try { streamData.put(Tuple2.of(row.mkString("\u0001"), eventTime)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }