/** * Create the transport client * @return */ private Client createTransportClient() { final String clusterName = indexFig.getClusterName(); final int port = indexFig.getPort(); ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName ) .put( "client.transport.sniff", true ); String nodeName = indexFig.getNodeName(); if ( "default".equals( nodeName ) ) { // no nodeName was specified, use hostname try { nodeName = InetAddress.getLocalHost().getHostName(); } catch ( UnknownHostException ex ) { nodeName = "client-" + RandomStringUtils.randomAlphabetic( 8 ); logger.warn( "Couldn't get hostname to use as ES node name, using {}", nodeName ); } } settings.put( "node.name", nodeName ); TransportClient transportClient = new TransportClient( settings.build() ); // we will connect to ES on all configured hosts for ( String host : indexFig.getHosts().split( "," ) ) { transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) ); } return transportClient; }
NodeBuilder builder = NodeBuilder.nodeBuilder(); Preconditions.checkArgument(config.has(INDEX_CONF_FILE) || config.has(INDEX_DIRECTORY), "Must either configure configuration file or base directory"); if (config.has(INDEX_CONF_FILE)) { String configFile = config.get(INDEX_CONF_FILE); ImmutableSettings.Builder sb = ImmutableSettings.settingsBuilder(); log.debug("Configuring ES from YML file [{}]", configFile); FileInputStream fis = null; try { fis = new FileInputStream(configFile); sb.loadFromStream(configFile, fis); builder.settings(sb.build()); } catch (FileNotFoundException e) { throw new TitanException(e); File f = new File(dataDirectory); if (!f.exists()) f.mkdirs(); ImmutableSettings.Builder b = ImmutableSettings.settingsBuilder(); for (String sub : DATA_SUBDIRS) { String subdir = dataDirectory + File.separator + sub; f = new File(subdir); if (!f.exists()) f.mkdirs(); b.put("path." + sub, subdir); b.put("script.disable_dynamic", false); b.put("indices.ttl.interval", "5s"); builder.settings(b.build());
@Activate protected void activate(final Map<String, Object> props) throws Exception { Map<String,String> stringProps = new HashMap<String,String>(); for (Map.Entry<String, Object> entry : props.entrySet()) { stringProps.put(entry.getKey(), entry.getValue().toString()); } Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); String configFilePath = stringProps.get("config"); if (configFilePath != null) { settings.loadFromUrl(new URL(configFilePath)); } settings.put(stringProps).classLoader(getClass().getClassLoader()); nodeDelegate = nodeBuilder().settings(settings).node(); }
Settings settings = settingsBuilder() .put(clientConfig) .put("http.enabled", false) .build(); node = nodeBuilder() .settings(settings) .client(true) .data(false) .node(); Settings settings = settingsBuilder() .put(clientConfig) .build(); TransportClient transportClient = new TransportClient(settings); for (TransportAddress transport: transportAddresses) { transportClient.addTransportAddress(transport); if (transportClient.connectedNodes().isEmpty()) { throw new RuntimeException("Elasticsearch client is not connected to any Elasticsearch nodes!");
@Override @SuppressWarnings("rawtypes") public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; String elasticSearchHost = (String) stormConf.get(StormElasticSearchConstants.ES_HOST); Integer elasticSearchPort = ((Long) stormConf.get(StormElasticSearchConstants.ES_PORT)).intValue(); String elasticSearchCluster = (String) stormConf.get(StormElasticSearchConstants.ES_CLUSTER_NAME); Boolean localMode = (Boolean) stormConf.get(ELASTIC_LOCAL_MODE); if (localMode != null && localMode) { client = nodeBuilder().local(true).node().client(); } else { Settings settings = ImmutableSettings.settingsBuilder().put("cluster.name", elasticSearchCluster).build(); client = new TransportClient(settings).addTransportAddress(new InetSocketTransportAddress( elasticSearchHost, elasticSearchPort)); } }
Settings settings = ImmutableSettings.settingsBuilder() .put( "cluster.name", clusterName ) .put( "discovery.zen.ping.unicast.hosts", hostString ) .put( "discovery.zen.ping.multicast.enabled", "false" ).put( "http.enabled", false ) .put( "client.transport.ping_timeout", 2000 ) // milliseconds .put( "client.transport.nodes_sampler_interval", 100 ).put( "network.tcp.blocking", true ) .put( "node.client", true ).put( "node.name", nodeName ) .build(); Node node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).data( false ).node();
@Override public Connection connect(Configuration config) throws IOException { log.debug("Configuring TransportClient"); ImmutableSettings.Builder settingsBuilder = settingsBuilder(config); if (config.has(ElasticSearchIndex.CLIENT_SNIFF)) { String k = "client.transport.sniff"; settingsBuilder.put(k, config.get(ElasticSearchIndex.CLIENT_SNIFF)); log.debug("Set {}: {}", k, config.get(ElasticSearchIndex.CLIENT_SNIFF)); } TransportClient tc = new TransportClient(settingsBuilder.build()); int defaultPort = config.has(INDEX_PORT) ? config.get(INDEX_PORT) : ElasticSearchIndex.HOST_PORT_DEFAULT; for (String host : config.get(INDEX_HOSTS)) { String[] hostparts = host.split(":"); String hostname = hostparts[0]; int hostport = defaultPort; if (hostparts.length == 2) hostport = Integer.parseInt(hostparts[1]); log.info("Configured remote host: {} : {}", hostname, hostport); tc.addTransportAddress(new InetSocketTransportAddress(hostname, hostport)); } return new Connection(null, tc); } },
/** * Open client to elaticsearch cluster * * @param clusterName */ private void openClient(String clusterName) { logger.info("Using ElasticSearch hostnames: {} ", Arrays.toString(serverAddresses)); Settings settings = ImmutableSettings.settingsBuilder() .put("cluster.name", clusterName).build(); TransportClient transportClient = new TransportClient(settings); for (InetSocketTransportAddress host : serverAddresses) { transportClient.addTransportAddress(host); } if (client != null) { client.close(); } client = transportClient; }
private Node buildNode(final String id, final Settings settings) { final String settingsSource = getClass().getName().replace('.', '/') + ".yml"; Settings finalSettings = settingsBuilder() .loadFromClasspath(settingsSource).put(defaultSettings) .put(settings).put("name", id).build(); if (finalSettings.get("gateway.type") == null) { // default to non gateway finalSettings = settingsBuilder().put(finalSettings) .put("gateway.type", "none").build(); } if (finalSettings.get("cluster.routing.schedule") != null) { // decrease the routing schedule so new nodes will be added quickly finalSettings = settingsBuilder().put(finalSettings) .put("cluster.routing.schedule", "50ms").build(); } final Node node = nodeBuilder().settings(finalSettings).build(); nodes.put(id, node); clients.put(id, node.client()); return node; }
@Override public Connection connect(Configuration config) throws IOException { log.debug("Configuring Node Client"); ImmutableSettings.Builder settingsBuilder = settingsBuilder(config); if (config.has(ElasticSearchIndex.TTL_INTERVAL)) { String k = "indices.ttl.interval"; settingsBuilder.put(k, config.get(ElasticSearchIndex.TTL_INTERVAL)); log.debug("Set {}: {}", k, config.get(ElasticSearchIndex.TTL_INTERVAL)); } makeLocalDirsIfNecessary(settingsBuilder, config); NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder().settings(settingsBuilder.build()); // Apply explicit Titan properties file overrides (otherwise conf-file or ES defaults apply) if (config.has(ElasticSearchIndex.CLIENT_ONLY)) { boolean clientOnly = config.get(ElasticSearchIndex.CLIENT_ONLY); nodeBuilder.client(clientOnly).data(!clientOnly); } if (config.has(ElasticSearchIndex.LOCAL_MODE)) nodeBuilder.local(config.get(ElasticSearchIndex.LOCAL_MODE)); if (config.has(ElasticSearchIndex.LOAD_DEFAULT_NODE_SETTINGS)) nodeBuilder.loadConfigSettings(config.get(ElasticSearchIndex.LOAD_DEFAULT_NODE_SETTINGS)); Node node = nodeBuilder.node(); Client client = node.client(); return new Connection(node, client); } };
/** * If ES already contains this instance's target index, then do nothing. * Otherwise, create the index, then wait {@link #CREATE_SLEEP}. * <p> * The {@code client} field must point to a live, connected client. * The {@code indexName} field must be non-null and point to the name * of the index to check for existence or create. * * @param config the config for this ElasticSearchIndex * @throws java.lang.IllegalArgumentException if the index could not be created */ private void checkForOrCreateIndex(Configuration config) { Preconditions.checkState(null != client); //Create index if it does not already exist IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(indexName)).actionGet(); if (!response.isExists()) { ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); ElasticSearchSetup.applySettingsFromTitanConf(settings, config, ES_CREATE_EXTRAS_NS); CreateIndexResponse create = client.admin().indices().prepareCreate(indexName) .setSettings(settings.build()).execute().actionGet(); try { final long sleep = config.get(CREATE_SLEEP); log.debug("Sleeping {} ms after {} index creation returned from actionGet()", sleep, indexName); Thread.sleep(sleep); } catch (InterruptedException e) { throw new TitanException("Interrupted while waiting for index to settle in", e); } if (!create.isAcknowledged()) throw new IllegalArgumentException("Could not create index: " + indexName); } }
public static Client getEsClientInstance() { if (ES_CLIENT_INSTANCE == null) { Settings settings = ImmutableSettings.settingsBuilder().put("discovery.zen.ping.multicast.enabled", false) .put("discovery.zen.ping.unicast.hosts", ES_HOST).put("discovery.zen.ping.unicast.enabled", true).build(); ES_CLIENT_INSTANCE = NodeBuilder.nodeBuilder().client(true).clusterName(ES_CLUSTER).local(false).settings(settings).node().client(); } return ES_CLIENT_INSTANCE; }
@Override protected UsageAwareDatastoreConnection<ElasticSearchDataContext> createDatastoreConnection() { final Builder settingsBuilder = ImmutableSettings.builder(); settingsBuilder.put("name", "AnalyzerBeans"); settingsBuilder.put("cluster.name", _clusterName); final Settings settings = settingsBuilder.build(); final TransportClient client = new TransportClient(settings); client.addTransportAddress(new InetSocketTransportAddress(_hostname, _port)); final ElasticSearchDataContext dataContext; if (_tableDefs == null || _tableDefs.length == 0) { dataContext = new ElasticSearchDataContext(client, _indexName); } else { dataContext = new ElasticSearchDataContext(client, _indexName, _tableDefs); } return new DatastoreConnectionImpl<ElasticSearchDataContext>(dataContext, this); }
private Node getNode() { String dataPath = esConfig.getDataPath(); ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder() .put("cluster.name", esConfig.getClusterName()) .put("http.enabled", true); if (!dataPath.isEmpty()) { settingsBuilder.put("path.data", dataPath); } Settings settings = settingsBuilder.build(); node = NodeBuilder.nodeBuilder() .settings(settings) .clusterName(esConfig.getClusterName()) .data(true).local(false).node(); logger.info("Created Embedded ElasticSearch node with cluster name {} and data location at {}", esConfig.getClusterName(), esConfig.getDataPath()); return node; }
@Test public void testIndexCreationOptions() throws InterruptedException, BackendException { final int shards = 77; ElasticsearchRunner esr = new ElasticsearchRunner(".", "indexCreationOptions.yml"); esr.start(); CommonsConfiguration cc = new CommonsConfiguration(new BaseConfiguration()); cc.set("index." + INDEX_NAME + ".elasticsearch.create.ext.number_of_shards", String.valueOf(shards)); cc.set("index." + INDEX_NAME + ".elasticsearch.ext.cluster.name", "indexCreationOptions"); ModifiableConfiguration config = new ModifiableConfiguration(GraphDatabaseConfiguration.ROOT_NS, cc, BasicConfiguration.Restriction.NONE); config.set(INTERFACE, ElasticSearchSetup.NODE.toString(), INDEX_NAME); Configuration indexConfig = config.restrictTo(INDEX_NAME); IndexProvider idx = new ElasticSearchIndex(indexConfig); simpleWriteAndQuery(idx); ImmutableSettings.Builder settingsBuilder = ImmutableSettings.settingsBuilder(); settingsBuilder.put("discovery.zen.ping.multicast.enabled", "false"); settingsBuilder.put("discovery.zen.ping.unicast.hosts", "localhost,127.0.0.1:9300"); settingsBuilder.put("cluster.name", "indexCreationOptions"); NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder().settings(settingsBuilder.build()); nodeBuilder.client(true).data(false).local(false); Node n = nodeBuilder.build().start(); GetSettingsResponse response = n.client().admin().indices().getSettings(new GetSettingsRequest().indices("titan")).actionGet(); assertEquals(String.valueOf(shards), response.getSetting("titan", "index.number_of_shards")); idx.close(); n.stop(); esr.stop(); }
Settings settings = ImmutableSettings.settingsBuilder() .put("name", shellSettings.settings().get(ShellSettings.NODE_NAME)) .put("http.enabled", false) .build(); Node node = NodeBuilder.nodeBuilder().clusterName(clusterName).client(true).settings(settings).build(); node.start();
public static Node createNode(String clusterName, boolean client, int port) { Settings settings = NodeBuilder.nodeBuilder().settings() .put("script.groovy.sandbox.enabled", true) .put("script.disable_dynamic", false) .put("transport.tcp.port", port).build(); Node node = NodeBuilder.nodeBuilder().client(client).data(!client).clusterName(clusterName).settings(settings).build(); node.start(); return node; } }
protected ShellNativeClient newTransportClient(TransportAddress... addresses) { Settings settings = ImmutableSettings.settingsBuilder().put("client.transport.ignore_cluster_name", true).build(); org.elasticsearch.client.transport.TransportClient client = new TransportClient(settings).addTransportAddresses(addresses); //if no connected node we can already close the (useless) client if (client.connectedNodes().size() == 0) { client.close(); return null; } AbstractClient<TransportClient, JsonInput, JsonOutput> shellClient = clientWrapper.wrapEsTransportClient(client); resourceRegistry.registerResource(shellClient); ShellNativeClient shellNativeClient = clientWrapper.wrapShellClient(shellClient); clientScopeSynchronizerRunner.startSynchronizer(shellNativeClient); return shellNativeClient; } }
public static TransportClient createTransportClient(String clusterName, InetSocketTransportAddress... addresses) { Settings settings = ImmutableSettings.settingsBuilder() .put("cluster.name", clusterName) .put("client.transport.sniff", true).build(); TransportClient transportClient = new TransportClient(settings).addTransportAddresses(addresses); return transportClient; }
private void internalCreateNode() { final NodeBuilder nodeBuilder = NodeBuilder.nodeBuilder(); if (null != configLocation) { internalLoadSettings(nodeBuilder, configLocation); } if (null != configLocations) { for (final Resource location : configLocations) { internalLoadSettings(nodeBuilder, location); } } if (null != settings) { nodeBuilder.getSettings().put(settings); } node = nodeBuilder.node(); }