private void doStart() { boolean error = true; try { channel.start(); sinkRunner.start(); sourceRunner.start(); supervisor.supervise(channel, new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START); supervisor.supervise(sinkRunner, new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START); supervisor.supervise(sourceRunner, new SupervisorPolicy.AlwaysRestartPolicy(), LifecycleState.START); error = false; } finally { if (error) { stopLogError(sourceRunner); stopLogError(channel); stopLogError(sinkRunner); supervisor.stop(); } } }
private void createSink() { sink = new ElasticSearchSink(); sink.setName("ElasticSearchSink-" + UUID.randomUUID()); channel = new MemoryChannel(); Map<String, String> channelParamters = new HashMap<>(); channelParamters.put("capacity", "100000"); channelParamters.put("transactionCapacity", "1000"); Context channelContext = new Context(channelParamters); Configurables.configure(channel, channelContext); channel.setName("ElasticSearchSinkChannel-" + UUID.randomUUID()); Map<String, String> paramters = new HashMap<>(); paramters.put(ElasticSearchSinkConstants.HOSTNAMES, "127.0.0.1:9310"); String indexNamePrefix = "recentlyviewed"; paramters.put(ElasticSearchSinkConstants.INDEX_NAME, indexNamePrefix); paramters.put(ElasticSearchSinkConstants.INDEX_TYPE, "clickevent"); paramters.put(ElasticSearchSinkConstants.CLUSTER_NAME, "jai-testclusterName"); paramters.put(ElasticSearchSinkConstants.BATCH_SIZE, "10"); paramters.put(ElasticSearchSinkConstants.SERIALIZER, ElasticSearchJsonBodyEventSerializer.class.getName()); Context sinkContext = new Context(paramters); sink.configure(sinkContext); sink.setChannel(channel); sink.start(); channel.start(); }
private void createSink() { sink = new HDFSEventSink(); sink.setName("HDFSEventSink-" + UUID.randomUUID()); channel = new MemoryChannel(); Map<String, String> channelParamters = new HashMap<>(); channelParamters.put("capacity", "100000"); channelParamters.put("transactionCapacity", "1000"); Context channelContext = new Context(channelParamters); Configurables.configure(channel, channelContext); channel.setName("HDFSEventSinkChannel-" + UUID.randomUUID()); Map<String, String> paramters = new HashMap<>(); paramters.put("hdfs.type", "hdfs"); String hdfsBasePath = hadoopClusterService.getHDFSUri() + "/searchevents"; paramters.put("hdfs.path", hdfsBasePath + "/%Y/%m/%d/%H"); paramters.put("hdfs.filePrefix", "searchevents"); paramters.put("hdfs.fileType", "DataStream"); paramters.put("hdfs.rollInterval", "0"); paramters.put("hdfs.rollSize", "0"); paramters.put("hdfs.idleTimeout", "1"); paramters.put("hdfs.rollCount", "0"); paramters.put("hdfs.batchSize", "1000"); paramters.put("hdfs.useLocalTimeStamp", "true"); Context sinkContext = new Context(paramters); sink.configure(sinkContext); sink.setChannel(channel); sink.start(); channel.start(); }
private void createSparkAvroSink() { sparkAvroChannel = new MemoryChannel(); Map<String, String> channelParamters = new HashMap<>(); channelParamters.put("capacity", "100000"); channelParamters.put("transactionCapacity", "1000"); Context channelContext = new Context(channelParamters); Configurables.configure(sparkAvroChannel, channelContext); String channelName = "SparkAvroMemoryChannel-" + UUID.randomUUID(); sparkAvroChannel.setName(channelName); sparkAvroSink = new AvroSink(); sparkAvroSink.setName("SparkAvroSink-" + UUID.randomUUID()); Map<String, String> paramters = new HashMap<>(); paramters.put("type", "avro"); paramters.put("hostname", "localhost"); paramters.put("port", "41111"); paramters.put("batch-size", "100"); Context sinkContext = new Context(paramters); sparkAvroSink.configure(sinkContext); Configurables.configure(sparkAvroSink, sinkContext); sparkAvroSink.setChannel(sparkAvroChannel); sparkAvroChannel.start(); sparkAvroSink.start(); }
channel.start(); avroSource.start();