joerg/elx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add index prune test, clean up test, update to xbib metrics 1.2.0

Browse source
This commit is contained in:
Jörg Prante 2019-02-24 23:52:06 +01:00
parent fa84c61336
commit 2b3ac518b5
29 changed files with 368 additions and 198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
build.gradle
View file

@ -6,7 +6,7 @@ plugins {
id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1" id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1"
} }
printf "Host: %s\nOS: %s %s %s\nJava: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" + printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" +
"Build: group: ${project.group} name: ${project.name} version: ${project.version}\n", "Build: group: ${project.group} name: ${project.name} version: ${project.version}\n",
InetAddress.getLocalHost(), InetAddress.getLocalHost(),
System.getProperty("os.name"), System.getProperty("os.name"),
@ -16,8 +16,9 @@ printf "Host: %s\nOS: %s %s %s\nJava: %s %s %s %s\nGradle: %s Groovy: %s Java: %
System.getProperty("java.vm.version"), System.getProperty("java.vm.version"),
System.getProperty("java.vm.vendor"), System.getProperty("java.vm.vendor"),
System.getProperty("java.vm.name"), System.getProperty("java.vm.name"),
gradle.gradleVersion, GroovySystem.getVersion(), JavaVersion.current() gradle.gradleVersion,
GroovySystem.getVersion(),
JavaVersion.current()
if (JavaVersion.current() < JavaVersion.VERSION_11) { if (JavaVersion.current() < JavaVersion.VERSION_11) {
throw new GradleException("This build must be run with java 11 or higher") throw new GradleException("This build must be run with java 11 or higher")
@ -92,24 +93,6 @@ subprojects {
} }
}*/ }*/
/*task javadocJar(type: Jar, dependsOn: classes) {
baseName "${rootProject.name}-${project.name}"
from javadoc
into "build/tmp"
classifier 'javadoc'
}
task sourcesJar(type: Jar, dependsOn: classes) {
baseName "${rootProject.name}-${project.name}"
from sourceSets.main.allSource
into "build/tmp"
classifier 'sources'
}
artifacts {
archives javadocJar, sourcesJar
}*/
task javadocJar(type: Jar, dependsOn: javadoc) { task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc' classifier 'javadoc'
} }

2
elx-api/build.gradle
View file

@ -1,4 +1,4 @@
dependencies { dependencies {
compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" compile "org.xbib:metrics-common:${project.property('xbib-metrics.version')}"
compile "org.xbib.elasticsearch:elasticsearch:${rootProject.property('elasticsearch-server.version')}" compile "org.xbib.elasticsearch:elasticsearch:${rootProject.property('elasticsearch-server.version')}"
} }

4
elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java
View file

@ -1,8 +1,8 @@
package org.xbib.elx.api; package org.xbib.elx.api;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.xbib.metrics.Count; import org.xbib.metrics.api.Count;
import org.xbib.metrics.Metered; import org.xbib.metrics.api.Metered;
import java.io.Closeable; import java.io.Closeable;

2
elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java
View file

@ -5,5 +5,5 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@FunctionalInterface @FunctionalInterface
public interface IndexAliasAdder { public interface IndexAliasAdder {
void addIndexAlias(IndicesAliasesRequest requwst, String index, String alias); void addIndexAlias(IndicesAliasesRequest request, String index, String alias);
} }

81
elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java
View file

@ -46,7 +46,7 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest
import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
@ -70,6 +70,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.sort.SortOrder;
@ -445,20 +446,21 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
ensureActive(); ensureActive();
ensureIndexGiven(index); ensureIndexGiven(index);
RecoveryRequest recoveryRequest = new RecoveryRequest(); GetSettingsRequest settingsRequest = new GetSettingsRequest();
recoveryRequest.indices(index); settingsRequest.indices(index);
recoveryRequest.activeOnly(true); GetSettingsResponse settingsResponse = client.execute(GetSettingsAction.INSTANCE, settingsRequest).actionGet();
RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, recoveryRequest).actionGet(); int shards = settingsResponse.getIndexToSettings().get(index).getAsInt("index.number_of_shards", -1);
int shards = response.getTotalShards(); if (shards > 0) {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
.indices(index) .indices(index)
.waitForActiveShards(shards).timeout(timeout); .waitForActiveShards(shards).timeout(timeout);
ClusterHealthResponse healthResponse = ClusterHealthResponse healthResponse =
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) { if (healthResponse != null && healthResponse.isTimedOut()) {
logger.error("timeout waiting for recovery"); logger.error("timeout waiting for recovery");
return false; return false;
}
} }
return true; return true;
} }
@ -736,7 +738,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
if (m2.matches()) { if (m2.matches()) {
Integer i2 = Integer.parseInt(m2.group(2)); Integer i2 = Integer.parseInt(m2.group(2));
int kept = candidateIndices.size() - indicesToDelete.size(); int kept = candidateIndices.size() - indicesToDelete.size();
if ((delta == 0 || (delta > 0 && i1 - i2 > delta)) && mintokeep <= kept) { if ((delta == 0 || (delta > 0 && i1 - i2 >= delta)) && mintokeep <= kept) {
indicesToDelete.add(s); indicesToDelete.add(s);
} }
} }
@ -755,13 +757,16 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
@Override @Override
public Long mostRecentDocument(String index, String timestampfieldname) { public Long mostRecentDocument(String index, String timestampfieldname) {
ensureActive(); ensureActive();
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
SearchResponse searchResponse = searchRequestBuilder.setIndices(index) SearchSourceBuilder builder = new SearchSourceBuilder();
.addStoredField(timestampfieldname) builder.sort(sort);
.setSize(1) builder.storedField(timestampfieldname);
.addSort(sort) builder.size(1);
.execute().actionGet(); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(index);
searchRequest.source(builder);
SearchResponse searchResponse =
client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
if (searchResponse.getHits().getHits().length == 1) { if (searchResponse.getHits().getHits().length == 1) {
SearchHit hit = searchResponse.getHits().getHits()[0]; SearchHit hit = searchResponse.getHits().getHits()[0];
if (hit.getFields().get(timestampfieldname) != null) { if (hit.getFields().get(timestampfieldname) != null) {
@ -903,13 +908,15 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { private void checkMapping(String index, String type, MappingMetaData mappingMetaData) {
try { try {
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); SearchSourceBuilder builder = new SearchSourceBuilder();
SearchResponse searchResponse = searchRequestBuilder.setSize(0) builder.query(QueryBuilders.matchAllQuery());
.setIndices(index) builder.size(0);
.setTypes(type) SearchRequest searchRequest = new SearchRequest();
.setQuery(QueryBuilders.matchAllQuery()) searchRequest.indices(index);
.execute() searchRequest.types(type);
.actionGet(); searchRequest.source(builder);
SearchResponse searchResponse =
client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
long total = searchResponse.getHits().getTotalHits(); long total = searchResponse.getHits().getTotalHits();
if (total > 0L) { if (total > 0L) {
Map<String, Long> fields = new TreeMap<>(); Map<String, Long> fields = new TreeMap<>();
@ -965,13 +972,15 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
} else if ("type".equals(key)) { } else if ("type".equals(key)) {
QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); QueryBuilder filterBuilder = QueryBuilders.existsQuery(path);
QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder);
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); SearchSourceBuilder builder = new SearchSourceBuilder();
SearchResponse searchResponse = searchRequestBuilder.setSize(0) builder.query(queryBuilder);
.setIndices(index) builder.size(0);
.setTypes(type) SearchRequest searchRequest = new SearchRequest();
.setQuery(queryBuilder) searchRequest.indices(index);
.execute() searchRequest.types(type);
.actionGet(); searchRequest.source(builder);
SearchResponse searchResponse =
client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
fields.put(path, searchResponse.getHits().getTotalHits()); fields.put(path, searchResponse.getHits().getTotalHits());
} }
} }

8
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java
View file

@ -2,10 +2,10 @@ package org.xbib.elx.common;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkMetric;
import org.xbib.metrics.Count; import org.xbib.metrics.api.Count;
import org.xbib.metrics.CountMetric; import org.xbib.metrics.api.Metered;
import org.xbib.metrics.Meter; import org.xbib.metrics.common.CountMetric;
import org.xbib.metrics.Metered; import org.xbib.metrics.common.Meter;
import java.util.concurrent.Executors; import java.util.concurrent.Executors;

8
elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java
View file

@ -9,7 +9,6 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
@ -71,9 +70,10 @@ public class AliasTest extends TestBase {
indicesAliasesRequest.addAliasAction(aliasAction); indicesAliasesRequest.addAliasAction(aliasAction);
client.admin().indices().aliases(indicesAliasesRequest).actionGet(); client.admin().indices().aliases(indicesAliasesRequest).actionGet();
GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesRequest getAliasesRequest = new GetAliasesRequest();
GetAliasesAction.INSTANCE); getAliasesRequest.aliases(alias);
GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); GetAliasesResponse getAliasesResponse =
client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
Set<String> result = new TreeSet<>(Collections.reverseOrder()); Set<String> result = new TreeSet<>(Collections.reverseOrder());
for (ObjectCursor<String> indexName : getAliasesResponse.getAliases().keys()) { for (ObjectCursor<String> indexName : getAliasesResponse.getAliases().keys()) {

2
elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java
View file

@ -21,7 +21,7 @@ public class ClusterBlockTest extends TestBase {
@Before @Before
public void startNodes() { public void startNodes() {
try { try {
setClusterName("test-cluster"); setClusterName("test-cluster-" + System.getProperty("user.name"));
startNode("1"); startNode("1");
// do not wait for green health state // do not wait for green health state
logger.info("ready"); logger.info("ready");

3
elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java
View file

@ -36,8 +36,7 @@ public class SearchTest extends TestBase {
.field("user9", "joerg") .field("user9", "joerg")
.field("rowcount", i) .field("rowcount", i)
.field("rs", 1234) .field("rs", 1234)
.endObject() .endObject());
);
builder.add(indexRequest); builder.add(indexRequest);
} }
client.bulk(builder.request()).actionGet(); client.bulk(builder.request()).actionGet();

55
elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java
View file

@ -3,17 +3,21 @@ package org.xbib.elx.common.test;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequestBuilder; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test; import org.junit.Test;
public class SimpleTest extends TestBase { public class SimpleTest extends TestBase {
@ -21,9 +25,9 @@ public class SimpleTest extends TestBase {
@Test @Test
public void test() throws Exception { public void test() throws Exception {
try { try {
DeleteIndexRequestBuilder deleteIndexRequestBuilder = DeleteIndexRequest deleteIndexRequest =
new DeleteIndexRequestBuilder(client("1"), DeleteIndexAction.INSTANCE, "test"); new DeleteIndexRequest().indices("test");
deleteIndexRequestBuilder.execute().actionGet(); client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
} catch (IndexNotFoundException e) { } catch (IndexNotFoundException e) {
// ignore if index not found // ignore if index not found
} }
@ -32,27 +36,24 @@ public class SimpleTest extends TestBase {
.put("index.analysis.analyzer.default.filter.1", "trim") .put("index.analysis.analyzer.default.filter.1", "trim")
.put("index.analysis.analyzer.default.tokenizer", "keyword") .put("index.analysis.analyzer.default.tokenizer", "keyword")
.build(); .build();
CreateIndexRequestBuilder createIndexRequestBuilder = new CreateIndexRequestBuilder(client("1"), CreateIndexAction.INSTANCE); CreateIndexRequest createIndexRequest = new CreateIndexRequest();
createIndexRequestBuilder.setIndex("test") createIndexRequest.index("test").settings(indexSettings);
.setSettings(indexSettings).execute().actionGet(); client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
IndexRequest indexRequest = new IndexRequest();
IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client("1"), IndexAction.INSTANCE); indexRequest.index("test").type("test").id("1")
indexRequestBuilder .source(XContentFactory.jsonBuilder().startObject().field("field",
.setIndex("test") "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject());
.setType("test") client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
.setId("1") RefreshRequest refreshRequest = new RefreshRequest();
.setSource(XContentFactory.jsonBuilder().startObject().field("field", refreshRequest.indices("test");
"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()) client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
.execute() SearchSourceBuilder builder = new SearchSourceBuilder();
.actionGet(); builder.query(QueryBuilders.matchQuery("field",
RefreshRequestBuilder refreshRequestBuilder = new RefreshRequestBuilder(client("1"), RefreshAction.INSTANCE); "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8"));
refreshRequestBuilder.setIndices("test").execute().actionGet(); SearchRequest searchRequest = new SearchRequest();
String doc = client("1").prepareSearch("test") searchRequest.indices("test").types("test");
.setTypes("test") searchRequest.source(builder);
.setQuery(QueryBuilders.matchQuery("field", String doc = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet()
"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8"))
.execute()
.actionGet()
.getHits().getAt(0).getSourceAsString(); .getHits().getAt(0).getSourceAsString();
assertEquals(doc, assertEquals(doc,

11
elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java
View file

@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.support.AbstractClient;
@ -59,7 +59,7 @@ public class TestBase {
public void startNodes() { public void startNodes() {
try { try {
logger.info("starting"); logger.info("starting");
setClusterName("test-cluster"); setClusterName("test-cluster-" + System.getProperty("user.name"));
startNode("1"); startNode("1");
findNodeAddress(); findNodeAddress();
try { try {
@ -73,9 +73,10 @@ public class TestBase {
} catch (ElasticsearchTimeoutException e) { } catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
} }
ClusterStateRequestBuilder clusterStateRequestBuilder = ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
new ClusterStateRequestBuilder(client("1"), ClusterStateAction.INSTANCE).all(); clusterStateRequest.all();
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port); logger.info("host = {} port = {}", host, port);
} catch (Throwable t) { } catch (Throwable t) {

19
elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java
View file

@ -1,11 +1,16 @@
package org.xbib.elx.common.test; package org.xbib.elx.common.test;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
@ -31,16 +36,20 @@ public class WildcardTest extends TestBase {
} }
private void index(Client client, String id, String fieldValue) throws IOException { private void index(Client client, String id, String fieldValue) throws IOException {
client.index(new IndexRequest("index", "type", id) client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id)
.source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())) .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject()))
.actionGet(); .actionGet();
client.admin().indices().refresh(new RefreshRequest()).actionGet(); client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet();
} }
private long count(Client client, QueryBuilder queryBuilder) { private long count(Client client, QueryBuilder queryBuilder) {
return client.prepareSearch("index").setTypes("type") SearchSourceBuilder builder = new SearchSourceBuilder();
.setQuery(queryBuilder) builder.query(queryBuilder);
.execute().actionGet().getHits().getTotalHits(); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("index");
searchRequest.types("type");
searchRequest.source(builder);
return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
} }
private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) {

2
elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java
View file

@ -1,6 +1,5 @@
package org.xbib.elx.node; package org.xbib.elx.node;
import io.netty.util.Version;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
@ -12,7 +11,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.node.Node; import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Utils;
import org.xbib.elx.common.AbstractExtendedClient; import org.xbib.elx.common.AbstractExtendedClient;
import java.io.IOException; import java.io.IOException;

8
elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java
View file

@ -32,7 +32,7 @@ import java.util.concurrent.TimeUnit;
public class ClientTest extends TestBase { public class ClientTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 25000L; private static final Long ACTIONS = 25000L;
@ -124,8 +124,6 @@ public class ClientTest extends TestBase {
client.waitForResponses(30L, TimeUnit.SECONDS); client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally { } finally {
assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
@ -149,7 +147,7 @@ public class ClientTest extends TestBase {
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedNodeClient client = ClientBuilder.builder(client("1")) final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
.provider(ExtendedNodeClientProvider.class) .provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build(); .build();
@ -187,8 +185,6 @@ public class ClientTest extends TestBase {
assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available"); logger.warn("skipping, no node available");
} catch (Exception e) {
logger.error(e.getMessage(), e);
} finally { } finally {
if (client.getBulkController().getLastBulkError() != null) { if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError()); logger.error("error", client.getBulkController().getLastBulkError());

18
elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java
View file

@ -3,8 +3,10 @@ package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test; import org.junit.Test;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters; import org.xbib.elx.common.Parameters;
@ -18,7 +20,7 @@ import static org.junit.Assert.*;
public class DuplicateIDTest extends TestBase { public class DuplicateIDTest extends TestBase {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
@ -39,11 +41,13 @@ public class DuplicateIDTest extends TestBase {
client.flush(); client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); client.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test"); client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) SearchSourceBuilder builder = new SearchSourceBuilder();
.setIndices("test") builder.query(QueryBuilders.matchAllQuery());
.setTypes("test") SearchRequest searchRequest = new SearchRequest();
.setQuery(matchAllQuery()); searchRequest.indices("test");
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); searchRequest.types("test");
searchRequest.source(builder);
long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
logger.info("hits = {}", hits); logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS); assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {

80
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java Normal file
View file

@ -0,0 +1,80 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class IndexPruneTest extends TestBase {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
@Test
public void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test1", settings);
client.shiftIndex("test", "test1", Collections.emptyList());
client.newIndex("test2", settings);
client.shiftIndex("test", "test2", Collections.emptyList());
client.newIndex("test3", settings);
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

2
elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java
View file

@ -21,7 +21,7 @@ import static org.junit.Assert.assertTrue;
public class IndexShiftTest extends TestBase { public class IndexShiftTest extends TestBase {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
@Test @Test
public void testIndexShift() throws Exception { public void testIndexShift() throws Exception {

2
elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java
View file

@ -30,7 +30,7 @@ import static org.junit.Assert.assertNull;
@Ignore @Ignore
public class ReplicaTest extends TestBase { public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName());
@Test @Test
public void testReplicaLevel() throws Exception { public void testReplicaLevel() throws Exception {

17
elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java
View file

@ -17,7 +17,7 @@ import static org.junit.Assert.assertNull;
public class SmokeTest extends TestBase { public class SmokeTest extends TestBase {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
@Test @Test
public void smokeTest() throws Exception { public void smokeTest() throws Exception {
@ -25,29 +25,26 @@ public class SmokeTest extends TestBase {
.provider(ExtendedNodeClientProvider.class) .provider(ExtendedNodeClientProvider.class)
.build(); .build();
try { try {
assertEquals(getClusterName(), client.getClusterName());
client.newIndex("test"); client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.delete("test", "1");
client.flush(); client.flush();
client.waitForResponses(30, TimeUnit.SECONDS); client.waitForResponses(30, TimeUnit.SECONDS);
assertEquals(getClusterName(), client.getClusterName());
client.checkMapping("test"); client.checkMapping("test");
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.flush();
client.waitForRecovery("test", 10L, TimeUnit.SECONDS);
client.delete("test", "1");
client.deleteIndex("test"); client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder() IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build()); .build());
assertEquals(0, indexDefinition.getReplicaLevel()); assertEquals(0, indexDefinition.getReplicaLevel());
client.newIndex(indexDefinition); client.newIndex(indexDefinition);
client.waitForRecovery(indexDefinition.getFullIndexName(), 30L, TimeUnit.SECONDS);
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush(); client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2); client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition); int replica = client.getReplicaLevel(indexDefinition);

14
elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java
View file

@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.support.AbstractClient;
@ -36,7 +36,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
public class TestBase { public class TestBase {
@ -50,8 +49,6 @@ public class TestBase {
private Map<String, AbstractClient> clients = new HashMap<>(); private Map<String, AbstractClient> clients = new HashMap<>();
private AtomicInteger counter = new AtomicInteger();
private String cluster; private String cluster;
private String host; private String host;
@ -62,7 +59,7 @@ public class TestBase {
public void startNodes() { public void startNodes() {
try { try {
logger.info("starting"); logger.info("starting");
this.cluster = "test-cluster-" + counter.incrementAndGet(); this.cluster = "test-cluster-" + System.getProperty("user.name");
startNode("1"); startNode("1");
findNodeAddress(); findNodeAddress();
try { try {
@ -76,9 +73,10 @@ public class TestBase {
} catch (ElasticsearchTimeoutException e) { } catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
} }
ClusterStateRequestBuilder clusterStateRequestBuilder = ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
new ClusterStateRequestBuilder(client("1"), ClusterStateAction.INSTANCE).all(); clusterStateRequest.all();
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port); logger.info("host = {} port = {}", host, port);
} catch (Throwable t) { } catch (Throwable t) {

2
elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java
View file

@ -108,7 +108,6 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
addresses.add(address); addresses.add(address);
} }
} }
logger.info("configured addresses = {}", addresses);
return addresses; return addresses;
} }
@ -116,7 +115,6 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
if (getClient() == null) { if (getClient() == null) {
throw new IllegalStateException("no client present"); throw new IllegalStateException("no client present");
} }
logger.debug("trying to connect to {}", addresses);
TransportClient transportClient = (TransportClient) getClient(); TransportClient transportClient = (TransportClient) getClient();
for (TransportAddress address : addresses) { for (TransportAddress address : addresses) {
transportClient.addTransportAddresses(address); transportClient.addTransportAddresses(address);

30
elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java
View file

@ -6,13 +6,15 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.Strings; import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
@ -32,7 +34,7 @@ import static org.junit.Assert.assertTrue;
public class ClientTest extends TestBase { public class ClientTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 25000L; private static final Long ACTIONS = 25000L;
@ -137,10 +139,14 @@ public class ClientTest extends TestBase {
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test"); client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) SearchSourceBuilder builder = new SearchSourceBuilder();
.setQuery(QueryBuilders.matchAllQuery()).setSize(0); builder.query(QueryBuilders.matchAllQuery());
assertEquals(numactions, builder.size(0);
searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test");
searchRequest.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertEquals(numactions, searchResponse.getHits().getTotalHits());
client.close(); client.close();
} }
} }
@ -200,10 +206,14 @@ public class ClientTest extends TestBase {
} }
assertNull(client.getBulkController().getLastBulkError()); assertNull(client.getBulkController().getLastBulkError());
client.refreshIndex("test"); client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) SearchSourceBuilder builder = new SearchSourceBuilder();
.setQuery(QueryBuilders.matchAllQuery()).setSize(0); builder.query(QueryBuilders.matchAllQuery());
assertEquals(maxthreads * actions, builder.size(0);
searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test");
searchRequest.source(builder);
SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits());
client.close(); client.close();
} }
} }

23
elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java
View file

@ -3,8 +3,11 @@ package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test; import org.junit.Test;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters; import org.xbib.elx.common.Parameters;
@ -13,14 +16,13 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
public class DuplicateIDTest extends TestBase { public class DuplicateIDTest extends TestBase {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
@ -42,11 +44,16 @@ public class DuplicateIDTest extends TestBase {
client.flush(); client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS); client.waitForResponses(30L, TimeUnit.SECONDS);
client.refreshIndex("test"); client.refreshIndex("test");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) SearchSourceBuilder builder = new SearchSourceBuilder();
.setIndices("test") builder.query(QueryBuilders.matchAllQuery());
.setTypes("test") builder.size(0);
.setQuery(matchAllQuery()); SearchRequest searchRequest = new SearchRequest();
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); searchRequest.indices("test");
searchRequest.types("test");
searchRequest.source(builder);
SearchResponse searchResponse =
client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("hits = {}", hits); logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS); assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) { } catch (NoNodeAvailableException e) {

81
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java Normal file
View file

@ -0,0 +1,81 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
public class IndexPruneTest extends TestBase {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
@Test
public void testPrune() throws IOException {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test1", settings);
client.shiftIndex("test", "test1", Collections.emptyList());
client.newIndex("test2", settings);
client.shiftIndex("test", "test2", Collections.emptyList());
client.newIndex("test3", settings);
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

2
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java
View file

@ -24,7 +24,7 @@ import static org.junit.Assert.assertTrue;
@Ignore @Ignore
public class IndexShiftTest extends TestBase { public class IndexShiftTest extends TestBase {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
@Test @Test
public void testIndexShift() throws Exception { public void testIndexShift() throws Exception {

30
elx-transport/src/test/java/org/xbib/elx/transport/test/ReplicaTest.java
View file

@ -6,13 +6,16 @@ import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats; import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.IndexingStats;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.ClientBuilder;
@ -23,14 +26,13 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
@Ignore @Ignore
public class ReplicaTest extends TestBase { public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName());
@Test @Test
public void testReplicaLevel() throws Exception { public void testReplicaLevel() throws Exception {
@ -73,15 +75,21 @@ public class ReplicaTest extends TestBase {
logger.info("refreshing"); logger.info("refreshing");
client.refreshIndex("test1"); client.refreshIndex("test1");
client.refreshIndex("test2"); client.refreshIndex("test2");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) SearchSourceBuilder builder = new SearchSourceBuilder();
.setIndices("test1", "test2") builder.query(QueryBuilders.matchAllQuery());
.setQuery(matchAllQuery()); builder.size(0);
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test1", "test2");
searchRequest.source(builder);
SearchResponse searchResponse =
client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("query total hits={}", hits); logger.info("query total hits={}", hits);
assertEquals(2468, hits); assertEquals(2468, hits);
IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE) IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
.all(); indicesStatsRequest.all();
IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); IndicesStatsResponse response =
client.getClient().execute(IndicesStatsAction.INSTANCE, indicesStatsRequest).actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) { for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue(); IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal(); CommonStats commonStats = indexStats.getTotal();

17
elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java
View file

@ -17,7 +17,7 @@ import static org.junit.Assert.assertNull;
public class SmokeTest extends TestBase { public class SmokeTest extends TestBase {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
@Test @Test
public void smokeTest() throws Exception { public void smokeTest() throws Exception {
@ -27,21 +27,14 @@ public class SmokeTest extends TestBase {
.put(getTransportSettings()) .put(getTransportSettings())
.build(); .build();
try { try {
assertEquals(getClusterName(), client.getClusterName());
client.newIndex("test"); client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.delete("test", "1");
client.flush(); client.flush();
client.waitForResponses(30, TimeUnit.SECONDS); client.waitForResponses(30, TimeUnit.SECONDS);
assertEquals(getClusterName(), client.getClusterName());
client.checkMapping("test"); client.checkMapping("test");
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.flush();
client.waitForRecovery("test", 10L, TimeUnit.SECONDS);
client.delete("test", "1");
client.deleteIndex("test"); client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder() IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
@ -50,8 +43,8 @@ public class SmokeTest extends TestBase {
client.newIndex(indexDefinition); client.newIndex(indexDefinition);
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush(); client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2); client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition); int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica); assertEquals(2, replica);

14
elx-transport/src/test/java/org/xbib/elx/transport/test/TestBase.java
View file

@ -9,7 +9,7 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin; import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.client.support.AbstractClient;
@ -36,7 +36,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Random; import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
public class TestBase { public class TestBase {
@ -50,8 +49,6 @@ public class TestBase {
private Map<String, AbstractClient> clients = new HashMap<>(); private Map<String, AbstractClient> clients = new HashMap<>();
private AtomicInteger counter = new AtomicInteger();
private String cluster; private String cluster;
private String host; private String host;
@ -62,7 +59,7 @@ public class TestBase {
public void startNodes() { public void startNodes() {
try { try {
logger.info("starting"); logger.info("starting");
this.cluster = "test-cluster-" + counter.incrementAndGet(); this.cluster = "test-cluster-" + System.getProperty("user.name");
startNode("1"); startNode("1");
findNodeAddress(); findNodeAddress();
try { try {
@ -76,9 +73,10 @@ public class TestBase {
} catch (ElasticsearchTimeoutException e) { } catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
} }
ClusterStateRequestBuilder clusterStateRequestBuilder = ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
new ClusterStateRequestBuilder(client("1"), ClusterStateAction.INSTANCE).all(); clusterStateRequest.all();
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port); logger.info("host = {} port = {}", host, port);
} catch (Throwable t) { } catch (Throwable t) {

4
gradle.properties
View file

@ -1,12 +1,12 @@
group = org.xbib group = org.xbib
name = elx name = elx
version = 6.3.2.0 version = 6.3.2.1
profile = default profile = default
release = 0 release = 0
elasticsearch-server.version = 6.3.2.2 elasticsearch-server.version = 6.3.2.2
log4j.version = 2.11.1 log4j.version = 2.11.1
xbib-metrics.version = 1.1.0 xbib-metrics.version = 1.2.0
xbib-netty-http.version = 4.1.33.0 xbib-netty-http.version = 4.1.33.0
# test # test