protected static Client getClient(ElasticsearchConfig config) { String name = config.getClientFactoryClassName(); try { Constructor c = Class.forName(name).getConstructor(ElasticsearchConfig.class); return ((ClientFactory) c.newInstance(config)).getClient(); } catch (InvocationTargetException | NoSuchMethodException | InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new RuntimeException(String.format("Could not instantiate class %s", name)); } }
public TransportClientFactory(ElasticsearchConfig config) { clientSettings = config.getElasticseachSettings(); if (config.getTransportHost().isPresent()) { transportHost = config.getTransportHost().get(); } else { throw new SamzaException("You must specify the transport host for TransportClientFactory" + "with the Elasticsearch system."); } if (config.getTransportPort().isPresent()) { transportPort = config.getTransportPort().get(); } else { throw new SamzaException("You must specify the transport port for TransportClientFactory" + "with the Elasticsearch system."); } }
protected static IndexRequestFactory getIndexRequestFactory(ElasticsearchConfig config) { if (config.getIndexRequestFactoryClassName().isPresent()) { return Util.getObj(config.getIndexRequestFactoryClassName().get(), IndexRequestFactory.class); } else { return new DefaultIndexRequestFactory(); } }
public Optional<Integer> getTransportPort() { if (containsKey(CONFIG_KEY_CLIENT_TRANSPORT_PORT)) { return Optional.of(getInt(CONFIG_KEY_CLIENT_TRANSPORT_PORT)); } else { return Optional.absent(); } }
public Optional<String> getTransportHost() { if (containsKey(CONFIG_KEY_CLIENT_TRANSPORT_HOST)) { return Optional.of(get(CONFIG_KEY_CLIENT_TRANSPORT_HOST)); } else { return Optional.absent(); } }
public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) { BulkProcessor.Builder builder = BulkProcessor.builder(client, listener); // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches. // This also means BulkProcessor#flush() is blocking as is also required. builder.setConcurrentRequests(0); if (config.getBulkFlushMaxActions().isPresent()) { builder.setBulkActions(config.getBulkFlushMaxActions().get()); } if (config.getBulkFlushMaxSizeMB().isPresent()) { builder.setBulkSize(new ByteSizeValue(config.getBulkFlushMaxSizeMB().get(), ByteSizeUnit.MB)); } if (config.getBulkFlushIntervalMS().isPresent()) { builder.setFlushInterval(TimeValue.timeValueMillis(config.getBulkFlushIntervalMS().get())); } return builder.build(); } }
private ElasticsearchConfig configForProperty(String key, String value) { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put(key, value); return new ElasticsearchConfig("es", new MapConfig(mapConfig)); }
public NodeClientFactory(ElasticsearchConfig config) { clientSettings = config.getElasticseachSettings(); }
@Test public void testGetBulkFlushMaxSizeMB() throws Exception { assertFalse(EMPTY_CONFIG.getBulkFlushMaxSizeMB().isPresent()); ElasticsearchConfig config = configForProperty("systems.es.bulk.flush.max.size.mb", "10"); assertTrue(config.getBulkFlushMaxSizeMB().isPresent()); assertEquals(Integer.valueOf(10), config.getBulkFlushMaxSizeMB().get()); }
@Test public void testGetTransportPort() throws Exception { assertFalse(EMPTY_CONFIG.getTransportPort().isPresent()); ElasticsearchConfig config = configForProperty("systems.es.client.transport.port", "9300"); assertTrue(config.getTransportPort().isPresent()); assertEquals(Integer.valueOf(9300), config.getTransportPort().get()); }
@Test public void testGetBulkFlushIntervalMS() throws Exception { assertFalse(EMPTY_CONFIG.getBulkFlushIntervalMS().isPresent()); ElasticsearchConfig config = configForProperty("systems.es.bulk.flush.interval.ms", "10"); assertTrue(config.getBulkFlushIntervalMS().isPresent()); assertEquals(Integer.valueOf(10), config.getBulkFlushIntervalMS().get()); }
@Test public void testGetTransportHost() throws Exception { assertFalse(EMPTY_CONFIG.getTransportHost().isPresent()); ElasticsearchConfig config = configForProperty("systems.es.client.transport.host", "example.org"); assertTrue(config.getTransportHost().isPresent()); assertEquals("example.org", config.getTransportHost().get()); }
@Test public void testGetBulkFlushMaxActions() throws Exception { assertFalse(EMPTY_CONFIG.getBulkFlushMaxActions().isPresent()); ElasticsearchConfig config = configForProperty("systems.es.bulk.flush.max.actions", "10"); assertEquals(Integer.valueOf(10), config.getBulkFlushMaxActions().get()); }
public Optional<Integer> getBulkFlushIntervalMS() { if (containsKey(CONFIG_KEY_BULK_FLUSH_INTERVALS_MS)) { return Optional.of(getInt(CONFIG_KEY_BULK_FLUSH_INTERVALS_MS)); } else { return Optional.absent(); } }
public Optional<String> getIndexRequestFactoryClassName() { if (containsKey(CONFIG_KEY_INDEX_REQUEST_FACTORY)) { return Optional.of(get(CONFIG_KEY_INDEX_REQUEST_FACTORY)); } else { return Optional.absent(); } }
@Override public SystemProducer getProducer(String name, Config config, MetricsRegistry metricsRegistry) { ElasticsearchConfig elasticsearchConfig = new ElasticsearchConfig(name, config); return new ElasticsearchSystemProducer(name, getBulkProcessorFactory(elasticsearchConfig), getClient(elasticsearchConfig), getIndexRequestFactory(elasticsearchConfig), new ElasticsearchSystemProducerMetrics(name, metricsRegistry)); }
@Test public void testGetElasticsearchSettings() throws Exception { ElasticsearchConfig config = configForProperty("systems.es.client.elasticsearch.foo", "bar"); assertEquals("bar", config.getElasticseachSettings().get("foo")); }
public Optional<Integer> getBulkFlushMaxActions() { if (containsKey(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS)) { return Optional.of(getInt(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS)); } else { return Optional.absent(); } }
public String getClientFactoryClassName() { if (containsKey(CONFIG_KEY_CLIENT_FACTORY)) { return get(CONFIG_KEY_CLIENT_FACTORY); } else { throw new SamzaException("You must specify a client factory class" + " for the Elasticsearch system."); } }
@Test public void testGetClientFactoryClassName() throws Exception { ElasticsearchConfig config = configForProperty("systems.es.client.factory", "bar"); assertEquals("bar", config.getClientFactoryClassName()); }