private static void issueWarnings(Client tc) { NodesInfoResponse nir = tc.admin().cluster().nodesInfo(new NodesInfoRequest()).actionGet(); Version maxVersion = nir.getNodes().stream().max((n1,n2) -> n1.getVersion().compareTo(n2.getVersion())).get().getVersion(); Version minVersion = nir.getNodes().stream().min((n1,n2) -> n1.getVersion().compareTo(n2.getVersion())).get().getVersion(); if(!maxVersion.equals(minVersion)) { System.out.println("WARNING: Your cluster consists of different node versions. It is not recommended to run sgadmin against a mixed cluster. This may fail."); System.out.println(" Minimum node version is "+minVersion.toString()); System.out.println(" Maximum node version is "+maxVersion.toString()); } else { System.out.println("Elasticsearch Version: "+minVersion.toString()); } if(nir.getNodes().size() > 0) { List<PluginInfo> pluginInfos = nir.getNodes().get(0).getPlugins().getPluginInfos(); String sgVersion = pluginInfos.stream().filter(p->p.getClassname().equals("com.floragunn.searchguard.SearchGuardPlugin")).map(p->p.getVersion()).findFirst().orElse("<unknown>"); System.out.println("Search Guard Version: "+sgVersion); } } }
private void deleteAllIndices() { ImmutableOpenMap<String, IndexMetaData> indices = elasticSearchClient.admin().cluster() .prepareState().get().getState() .getMetaData().getIndices(); indices.forEach(cursor -> { try { elasticSearchClient.admin() .indices() .delete(new DeleteIndexRequest(cursor.value.getIndex().getName())) .get(); } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } }); }
IndicesExistsRequestBuilder indicesExistBld = admin.indices().prepareExists( tempMeta.getIndex() ); IndicesExistsResponse indicesExistResponse = indicesExistBld.execute().get(); if ( !indicesExistResponse.isExists() ) { showError( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.Error.NoIndex" ) ); RecoveryRequestBuilder indicesBld = admin.indices().prepareRecoveries( tempMeta.getIndex() ); ActionFuture<RecoveryResponse> lafInd = indicesBld.execute(); String shards = "" + lafInd.get().getSuccessfulShards() + "/" + lafInd.get().getTotalShards(); showMessage( BaseMessages.getString( PKG, "ElasticSearchBulkDialog.TestIndex.TestOK", shards ) ); break; case CLUSTER: ClusterStateRequestBuilder clusterBld = admin.cluster().prepareState(); ActionFuture<ClusterStateResponse> lafClu = clusterBld.execute(); ClusterStateResponse cluResp = lafClu.actionGet(); String name = cluResp.getClusterName().value(); ClusterState cluState = cluResp.getState();
TransportClient tClient = TransportClient.builder().settings(settings).build(); for (String h : nodeList) { String[] nodes = h.split(":"); try { tClient.addTransportAddress(new InetSocketTransportAddress( InetAddress.getByName(nodes[0]), Integer.parseInt(nodes[1]) client.admin().indices() .exists(Requests.indicesExistsRequest(indexKey)).actionGet() .isExists(); if (exists && newdb) { client.admin().indices().prepareDelete(indexKey).execute().actionGet(); client.admin().indices().create( new CreateIndexRequest(indexKey) .settings( .put("index.number_of_replicas", numberOfReplicas) .put("index.mapping._id.indexed", true) )).actionGet(); client.admin().cluster().health(new ClusterHealthRequest().waitForGreenStatus()).actionGet();
@Override public Set<String> getFolderNames() throws IOException, MessagingException { createIndexIfNotExists(); client.admin().indices().refresh(new RefreshRequest()).actionGet(); final HashSet<String> uids = new HashSet<String>(); SearchResponse scrollResp = client.prepareSearch().setIndices(index).setTypes(type).setSearchType(SearchType.SCAN) .setQuery(QueryBuilders.matchAllQuery()).addField("folderFullName").setScroll(new TimeValue(1000)).setSize(1000).execute() .actionGet(); while (true) { scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(1000)).execute().actionGet(); boolean hitsRead = false; for (final SearchHit hit : scrollResp.getHits()) { hitsRead = true; uids.add((String) hit.getFields().get("folderFullName").getValue()); } if (!hitsRead) { break; } } if (logger.isDebugEnabled()) { logger.debug("Currently locally stored folders: {}", uids); } return uids; }
/** * If ES already contains this instance's target index, then do nothing. * Otherwise, create the index, then wait {@link #CREATE_SLEEP}. * <p> * The {@code client} field must point to a live, connected client. * The {@code indexName} field must be non-null and point to the name * of the index to check for existence or create. * * @param config the config for this ElasticSearchIndex * @throws java.lang.IllegalArgumentException if the index could not be created */ private void checkForOrCreateIndex(Configuration config) { Preconditions.checkState(null != client); //Create index if it does not already exist IndicesExistsResponse response = client.admin().indices().exists(new IndicesExistsRequest(indexName)).actionGet(); if (!response.isExists()) { ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder(); ElasticSearchSetup.applySettingsFromTitanConf(settings, config, ES_CREATE_EXTRAS_NS); CreateIndexResponse create = client.admin().indices().prepareCreate(indexName) .setSettings(settings.build()).execute().actionGet(); try { final long sleep = config.get(CREATE_SLEEP); log.debug("Sleeping {} ms after {} index creation returned from actionGet()", sleep, indexName); Thread.sleep(sleep); } catch (InterruptedException e) { throw new TitanException("Interrupted while waiting for index to settle in", e); } if (!create.isAcknowledged()) throw new IllegalArgumentException("Could not create index: " + indexName); } }
final WhoAmIResponse whoAmIRes = tc.execute(WhoAmIAction.INSTANCE, new WhoAmIRequest()).actionGet(); sb.append(Strings.toString(whoAmIRes,true, true)); } catch (Exception e1) { LicenseInfoResponse res = tc.execute(LicenseInfoAction.INSTANCE, new LicenseInfoRequest()).actionGet(); sb.append(Strings.toString(res,true, true)); } catch (Exception e1) { ClusterHealthResponse nir = tc.admin().cluster().health(new ClusterHealthRequest()).actionGet(); sb.append(Strings.toString(nir,true, true)); } catch (Exception e1) { NodesInfoResponse nir = tc.admin().cluster().nodesInfo(new NodesInfoRequest()).actionGet(); sb.append(Strings.toString(nir,true, true)); } catch (Exception e1) { NodesStatsResponse nir = tc.admin().cluster().nodesStats(new NodesStatsRequest()).actionGet(); sb.append(Strings.toString(nir,true, true)); } catch (Exception e1) { PendingClusterTasksResponse nir = tc.admin().cluster().pendingClusterTasks(new PendingClusterTasksRequest()).actionGet(); sb.append(Strings.toString(nir,true, true)); } catch (Exception e1) { IndicesStatsResponse nir = tc.admin().indices().stats(new IndicesStatsRequest()).actionGet(); sb.append(Strings.toString(nir, true, true)); } catch (Exception e1) {
private List<SearchHit> scrollTillLimit(TableInJoinRequestBuilder tableInJoinRequest, Integer hintLimit) { SearchResponse scrollResp = scrollOneTimeWithMax(client,tableInJoinRequest); updateMetaSearchResults(scrollResp); List<SearchHit> hitsWithScan = new ArrayList<>(); int curentNumOfResults = 0; SearchHit[] hits = scrollResp.getHits().getHits(); if (hintLimit == null) hintLimit = MAX_RESULTS_FOR_FIRST_TABLE; while (hits.length != 0 && curentNumOfResults < hintLimit) { curentNumOfResults += hits.length; Collections.addAll(hitsWithScan, hits); if (curentNumOfResults >= MAX_RESULTS_FOR_FIRST_TABLE) { //todo: log or exception? System.out.println("too many results for first table, stoping at:" + curentNumOfResults); break; } scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits(); } return hitsWithScan; }
@Override public long getCount() { long count = 0; IndicesAdminClient admin = elastic.getClient().admin().indices(); IndicesExistsRequestBuilder builder = admin.prepareExists(config.getMisc().getDindex().getName()); if (builder.execute().actionGet().isExists()) { SearchResponse countResponse = elastic.getClient().prepareSearch(config.getMisc().getDindex().getName()) .setTypes(config.getMisc().getCtype().getName()) .setSearchType(SearchType.QUERY_THEN_FETCH) .setSize(0) .execute().actionGet(); count = countResponse.getHits().getTotalHits(); } else { logger.info("Index/Type does not exist or does not contain the record"); System.exit(-1); } logger.info("Elastic Index/Type count: " + count); return count; }
IndicesExistsResponse response = client.admin().indices() .exists(new IndicesExistsRequest(indexName)).actionGet(); client.admin().indices().prepareDelete(this.indexName).execute() .actionGet(); forceCreate = true; CreateIndexResponse create = client.admin().indices() .prepareCreate(indexName).execute().actionGet(); try { Thread.sleep(200);
tc.index(new IndexRequest("esb-prod-1").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("esb-prod-2").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":2}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("esb-prod-3").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":3}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("esb-prod-4").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":4}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("esb-prod-5").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":5}", XContentType.JSON)).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("esb-prod-1","esb-prod-2","esb-prod-3","esb-prod-4","esb-prod-5").alias("esb-prod-all"))).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("esb-prod-1").alias("esb-alias-1"))).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("esb-prod-2").alias("esb-alias-2"))).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("esb-prod-3").alias("esb-alias-3"))).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("esb-prod-4").alias("esb-alias-4"))).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("esb-prod-5").alias("esb-alias-5"))).actionGet();
@Override public void clearStorage() throws BackendException { try { try { client.admin().indices() .delete(new DeleteIndexRequest(indexName)).actionGet(); // We wait for one second to let ES delete the river Thread.sleep(1000); } catch (IndexMissingException e) { // Index does not exist... Fine } } catch (Exception e) { throw new PermanentBackendException("Could not delete index " + indexName, e); } finally { close(); } }
tc.addTransportAddress(new TransportAddress(new InetSocketAddress(info.nodeHost, info.nodePort))); Assert.assertEquals(info.numNodes, tc.admin().cluster().nodesInfo(new NodesInfoRequest()).actionGet().getNodes().size()); tc.admin().indices().create(new CreateIndexRequest("searchguard")).actionGet(); } catch (Exception e) { tc.index(ir).actionGet(); .execute(ConfigUpdateAction.INSTANCE, new ConfigUpdateRequest(ConfigConstants.CONFIG_NAMES.toArray(new String[0]))) .actionGet(); Assert.assertEquals(info.numNodes, cur.getNodes().size()); SearchResponse sr = tc.search(new SearchRequest("searchguard")).actionGet(); sr = tc.search(new SearchRequest("searchguard")).actionGet(); Assert.assertTrue(tc.get(new GetRequest("searchguard", "sg", "config")).actionGet().isExists()); Assert.assertTrue(tc.get(new GetRequest("searchguard","sg","internalusers")).actionGet().isExists()); Assert.assertTrue(tc.get(new GetRequest("searchguard","sg","roles")).actionGet().isExists()); Assert.assertTrue(tc.get(new GetRequest("searchguard","sg","rolesmapping")).actionGet().isExists()); Assert.assertTrue(tc.get(new GetRequest("searchguard","sg","actiongroups")).actionGet().isExists()); Assert.assertFalse(tc.get(new GetRequest("searchguard","sg","rolesmapping_xcvdnghtu165759i99465")).actionGet().isExists()); Assert.assertTrue(tc.get(new GetRequest("searchguard","sg","config")).actionGet().isExists());
public synchronized void deleteIndex() { DeleteIndexResponse response = client.admin().indices().delete(new DeleteIndexRequest(index)).actionGet(); if (!response.isAcknowledged()) { throw new RuntimeException("cannot delete index " + index); } client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); }
@Test public void testAdvancedMapping() throws Exception { setup(Settings.EMPTY, new DynamicSgConfig(), Settings.EMPTY, true, ClusterConfiguration.DEFAULT); try (TransportClient tc = getInternalTransportClient(this.clusterInfo, Settings.EMPTY)) { tc.admin().indices().create(new CreateIndexRequest("myindex1") .mapping("mytype1", FileHelper.loadFile("mapping1.json"), XContentType.JSON)).actionGet(); tc.admin().indices().create(new CreateIndexRequest("myindex2") .mapping("mytype2", FileHelper.loadFile("mapping2.json"), XContentType.JSON)).actionGet(); tc.admin().indices().create(new CreateIndexRequest("myindex3") .mapping("mytype3", FileHelper.loadFile("mapping3.json"), XContentType.JSON)).actionGet(); tc.admin().indices().create(new CreateIndexRequest("myindex4") .mapping("mytype4", FileHelper.loadFile("mapping4.json"), XContentType.JSON)).actionGet(); } RestHelper rh = nonSslRestHelper(); System.out.println("############ write into mapping 1"); String data1 = FileHelper.loadFile("data1.json"); System.out.println(rh.executePutRequest("myindex1/mytype1/1?refresh", data1, encodeBasicHeader("nagilum", "nagilum"))); System.out.println(rh.executePutRequest("myindex1/mytype1/1?refresh", data1, encodeBasicHeader("nagilum", "nagilum"))); System.out.println("############ write into mapping 2"); System.out.println(rh.executePutRequest("myindex2/mytype2/2?refresh", data1, encodeBasicHeader("nagilum", "nagilum"))); System.out.println(rh.executePutRequest("myindex2/mytype2/2?refresh", data1, encodeBasicHeader("nagilum", "nagilum"))); System.out.println("############ write into mapping 3"); String parent = FileHelper.loadFile("data2.json"); String child = FileHelper.loadFile("data3.json"); System.out.println(rh.executePutRequest("myindex3/mytype3/1?refresh", parent, encodeBasicHeader("nagilum", "nagilum"))); System.out.println(rh.executePutRequest("myindex3/mytype3/2?routing=1&refresh", child, encodeBasicHeader("nagilum", "nagilum"))); System.out.println("############ write into mapping 4"); System.out.println(rh.executePutRequest("myindex4/mytype4/1?refresh", parent, encodeBasicHeader("nagilum", "nagilum"))); System.out.println(rh.executePutRequest("myindex4/mytype4/2?routing=1&refresh", child, encodeBasicHeader("nagilum", "nagilum"))); }
private synchronized void createIndex() { CreateIndexResponse response = client.admin().indices().create(new CreateIndexRequest(index)).actionGet(); if (!response.isAcknowledged()) { throw new RuntimeException("cannot create index " + index); } client.admin().cluster().prepareHealth().setWaitForYellowStatus().execute().actionGet(); }
private void cleanOrphanIndex() { String[] indices = client.admin().cluster().prepareState().execute().actionGet().getState() .getMetaData().getConcreteAllIndices(); List<String> orphans = Stream.of(indices) .filter(i -> i.startsWith(this.configuration.getAlias())) .filter(i -> 0 == client.admin().indices().getAliases(new GetAliasesRequest().indices(i)) .actionGet().getAliases().size()) .collect(Collectors.toList()); for (String orphan : orphans) { client.admin().indices().delete(new DeleteIndexRequest(orphan)).actionGet(); } }
@BeforeClass public void prepareTestPersistUpdater() throws Exception { testConfiguration = new ComponentConfigurator<>(ElasticsearchWriterConfiguration.class).detectConfiguration("ElasticsearchPersistUpdaterIT"); testClient = ElasticsearchClientManager.getInstance(testConfiguration).client(); ClusterHealthRequest clusterHealthRequest = Requests.clusterHealthRequest(); ClusterHealthResponse clusterHealthResponse = testClient.admin().cluster().health(clusterHealthRequest).actionGet(); assertNotEquals(clusterHealthResponse.getStatus(), ClusterHealthStatus.RED); IndicesExistsRequest indicesExistsRequest = Requests.indicesExistsRequest(testConfiguration.getIndex()); IndicesExistsResponse indicesExistsResponse = testClient.admin().indices().exists(indicesExistsRequest).actionGet(); assertTrue(indicesExistsResponse.isExists()); }
private String[] getIndexesFromEs(final String aliasName){ final AdminClient adminClient = this.provider.getClient().admin(); //remove write alias, can only have one ImmutableOpenMap<String, List<AliasMetaData>> aliasMap = adminClient.indices().getAliases( new GetAliasesRequest( aliasName ) ).actionGet().getAliases(); return aliasMap.keys().toArray( String.class ); }