joerg/elx
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update to Junit 5.4.2, ES 6.3.2.3, gradle 5.3, fix tests

Browse source
This commit is contained in:
Jörg Prante 2019-05-03 22:50:13 +02:00
parent 2b3ac518b5
commit d4e40145be
47 changed files with 1587 additions and 1717 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
build.gradle
View file

@ -21,7 +21,7 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s
JavaVersion.current()
if (JavaVersion.current() < JavaVersion.VERSION_11) {
throw new GradleException("This build must be run with java 11 or higher")
throw new GradleException("This build must be run with java 11")
}
subprojects {
@ -39,9 +39,11 @@ subprojects {
}
dependencies {
testCompile "junit:junit:${project.property('junit.version')}"
testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}"
testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}"
testImplementation "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}"
testImplementation "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}"
testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}"
testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}"
asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}"
wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}"
}
@ -63,21 +65,35 @@ subprojects {
}
test {
jvmArgs =[
'--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED',
'--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED',
'--add-opens=java.base/java.nio=ALL-UNNAMED'
]
enabled = true
useJUnitPlatform()
// we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery
doFirst {
jvmArgs = [
'--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED',
'--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED',
'--add-opens=java.base/java.nio=ALL-UNNAMED'
]
}
systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager'
systemProperty 'path.home', "${project.buildDir}"
systemProperty 'jna.debug_load', 'true'
testLogging {
showStandardStreams = true
exceptionFormat = 'full'
events 'PASSED', 'FAILED', 'SKIPPED'
}
afterSuite { desc, result ->
if (!desc.parent) {
println "\nTest result: ${result.resultType}"
println "Test summary: ${result.testCount} tests, " +
"${result.successfulTestCount} succeeded, " +
"${result.failedTestCount} failed, " +
"${result.skippedTestCount} skipped"
}
}
}
clean {
delete "data"
delete "logs"
delete "out"
}
@ -94,12 +110,12 @@ subprojects {
}*/
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
archiveClassifier.set('javadoc')
}
task sourcesJar(type: Jar, dependsOn: classes) {
from sourceSets.main.allSource
classifier 'sources'
archiveClassifier.set('sources')
}
artifacts {
@ -164,35 +180,3 @@ subprojects {
}
}
}
/*asciidoctor {
attributes toc: 'left',
doctype: 'book',
icons: 'font',
encoding: 'utf-8',
sectlink: true,
sectanchors: true,
linkattrs: true,
imagesdir: 'img',
'source-highlighter': 'coderay'
}*/
/*
task aggregatedJavadoc(type: Javadoc) {
group = 'aggregation'
description = 'Generates aggregated Javadoc API documentation.'
title = "$description $version API"
destinationDir = file("$buildDir/docs/javadoc")
def sourceProjects = subprojects.findAll {
it.plugins.hasPlugin('java') || it.plugins.hasPlugin('groovy')
}
source sourceProjects.collect {
it.sourceSets.main.allJava
}
classpath = files(sourceProjects.collect {
it.sourceSets.main.runtimeClasspath
})
//options.overview = 'gradle/api/overview.html'
options.showFromProtected()
}
*/

4
elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java
View file

@ -398,9 +398,9 @@ public interface ExtendedClient extends Flushable, Closeable {
String resolveMostRecentIndex(String alias);
/**
* Get all index filters.
* Get all index aliases.
* @param index the index
* @return map of index filters
* @return map of index aliases
*/
Map<String, String> getAliases(String index);

25
elx-api/src/main/java/org/xbib/elx/api/ReadClient.java Normal file
View file

@ -0,0 +1,25 @@
package org.xbib.elx.api;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
public interface ReadClient {
ActionFuture<GetResponse> get(GetRequest getRequest);
void get(GetRequest request, ActionListener<GetResponse> listener);
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
ActionFuture<SearchResponse> search(SearchRequest request);
void search(SearchRequest request, ActionListener<SearchResponse> listener);
}

6
elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java Normal file
View file

@ -0,0 +1,6 @@
package org.xbib.elx.api;
public interface ReadClientProvider<C extends ReadClient> {
C getReadClient();
}

3
elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java
View file

@ -454,7 +454,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
.indices(index)
.waitForActiveShards(shards).timeout(timeout);
.waitForActiveShards(shards)
.timeout(timeout);
ClusterHealthResponse healthResponse =
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {

23
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java
View file

@ -8,7 +8,6 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
@ -83,12 +82,12 @@ public class DefaultBulkController implements BulkController {
maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest);
}
this.bulkListener = new BulkListener();
DefaultBulkProcessor.Builder builder = DefaultBulkProcessor.builder((Client) client.getClient(), bulkListener)
this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener)
.setBulkActions(maxActionsPerRequest)
.setConcurrentRequests(maxConcurrentRequests)
.setFlushInterval(flushIngestInterval)
.setBulkSize(maxVolumePerRequest);
this.bulkProcessor = builder.build();
.setBulkSize(maxVolumePerRequest)
.build();
this.active.set(true);
}
@ -115,9 +114,7 @@ public class DefaultBulkController implements BulkController {
@Override
public void index(IndexRequest indexRequest) {
if (!active.get()) {
throw new IllegalStateException("inactive");
}
ensureActiveAndBulk();
try {
if (bulkMetric != null) {
bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
@ -226,6 +223,18 @@ public class DefaultBulkController implements BulkController {
}
}
private void ensureActiveAndBulk() {
if (!active.get()) {
throw new IllegalStateException("inactive");
}
if (bulkProcessor == null) {
throw new UnsupportedOperationException("bulk processor not present");
}
if (bulkListener == null) {
throw new UnsupportedOperationException("bulk listener not present");
}
}
private class BulkListener implements DefaultBulkProcessor.Listener {
private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener");

42
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java
View file

@ -8,7 +8,8 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
@ -16,6 +17,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.xbib.elx.api.BulkProcessor;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@ -47,7 +49,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
private volatile boolean closed;
private DefaultBulkProcessor(Client client, Listener listener, String name, int concurrentRequests,
private DefaultBulkProcessor(ElasticsearchClient client, Listener listener, String name, int concurrentRequests,
int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) {
this.executionIdGen = new AtomicLong();
this.closed = false;
@ -59,7 +61,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
new AsyncBulkRequestHandler(client, listener, concurrentRequests);
if (flushInterval != null) {
this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,
EsExecutors.daemonThreadFactory(client.settings(),
EsExecutors.daemonThreadFactory(Settings.EMPTY,
name != null ? "[" + name + "]" : "" + "bulk_processor"));
this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
@ -71,10 +73,9 @@ public class DefaultBulkProcessor implements BulkProcessor {
}
}
public static Builder builder(Client client, Listener listener) {
if (client == null) {
throw new NullPointerException("The client you specified while building a BulkProcessor is null");
}
public static Builder builder(ElasticsearchClient client, Listener listener) {
Objects.requireNonNull(client, "The client you specified while building a BulkProcessor is null");
Objects.requireNonNull(listener, "A listener for the BulkProcessor is required but null");
return new Builder(client, listener);
}
@ -87,6 +88,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
*/
@Override
public synchronized boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException {
Objects.requireNonNull(unit, "A time unit is required for awaitFlush() but null");
if (closed) {
return true;
}
@ -95,7 +97,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
execute();
}
// wait for all bulk responses
return this.bulkRequestHandler.close(timeout, unit);
return bulkRequestHandler.close(timeout, unit);
}
/**
@ -115,18 +117,19 @@ public class DefaultBulkProcessor implements BulkProcessor {
*/
@Override
public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
Objects.requireNonNull(unit, "A time unit is required for awaitCLose() but null");
if (closed) {
return true;
}
closed = true;
if (this.scheduledFuture != null) {
FutureUtils.cancel(this.scheduledFuture);
if (scheduledFuture != null) {
FutureUtils.cancel(scheduledFuture);
this.scheduler.shutdown();
}
if (bulkRequest.numberOfActions() > 0) {
execute();
}
return this.bulkRequestHandler.close(timeout, unit);
return bulkRequestHandler.close(timeout, unit);
}
/**
@ -215,8 +218,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
private boolean isOverTheLimit() {
return bulkActions != -1 &&
bulkRequest.numberOfActions() >= bulkActions ||
bulkSize != -1 &&
bulkRequest.estimatedSizeInBytes() >= bulkSize;
bulkSize != -1 && bulkRequest.estimatedSizeInBytes() >= bulkSize;
}
/**
@ -224,7 +226,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
*/
public static class Builder {
private final Client client;
private final ElasticsearchClient client;
private final Listener listener;
@ -245,7 +247,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
* @param client the client
* @param listener the listener
*/
Builder(Client client, Listener listener) {
Builder(ElasticsearchClient client, Listener listener) {
this.client = client;
this.listener = listener;
}
@ -339,11 +341,12 @@ public class DefaultBulkProcessor implements BulkProcessor {
private static class SyncBulkRequestHandler implements BulkRequestHandler {
private final Client client;
private final ElasticsearchClient client;
private final DefaultBulkProcessor.Listener listener;
SyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener) {
SyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener) {
Objects.requireNonNull(listener, "A listener is required for SyncBulkRequestHandler but null");
this.client = client;
this.listener = listener;
}
@ -371,7 +374,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
private static class AsyncBulkRequestHandler implements BulkRequestHandler {
private final Client client;
private final ElasticsearchClient client;
private final DefaultBulkProcessor.Listener listener;
@ -379,7 +382,8 @@ public class DefaultBulkProcessor implements BulkProcessor {
private final int concurrentRequests;
private AsyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener, int concurrentRequests) {
private AsyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener, int concurrentRequests) {
Objects.requireNonNull(listener, "A listener is required for AsyncBulkRequestHandler but null");
this.client = client;
this.listener = listener;
this.concurrentRequests = concurrentRequests;

49
elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java
View file

@ -1,19 +1,19 @@
package org.xbib.elx.common.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.Strings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import java.util.Collections;
import java.util.Iterator;
@ -22,44 +22,54 @@ import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class AliasTest extends TestBase {
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class AliasTest {
private static final Logger logger = LogManager.getLogger(AliasTest.class.getName());
private final TestExtension.Helper helper;
AliasTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testAlias() {
Client client = client("1");
void testAlias() {
ElasticsearchClient client = helper.client("1");
CreateIndexRequest indexRequest = new CreateIndexRequest("test");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
// put alias
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
String[] indices = new String[]{"test"};
String[] aliases = new String[]{"test_alias"};
String[] indices = new String[] { "test" };
String[] aliases = new String[] { "test_alias" };
IndicesAliasesRequest.AliasActions aliasAction =
new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
.indices(indices)
.aliases(aliases);
indicesAliasesRequest.addAliasAction(aliasAction);
client.admin().indices().aliases(indicesAliasesRequest).actionGet();
client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
// get alias
GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY);
long t0 = System.nanoTime();
GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(getAliasesRequest).actionGet();
GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
long t1 = (System.nanoTime() - t0) / 1000000;
logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1);
assertTrue(t1 >= 0);
}
@Test
public void testMostRecentIndex() {
Client client = client("1");
void testMostRecentIndex() {
ElasticsearchClient client = helper.client("1");
String alias = "test";
CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
indexRequest = new CreateIndexRequest("test20160102");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
indexRequest = new CreateIndexRequest("test20160103");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
String[] indices = new String[]{"test20160101", "test20160102", "test20160103"};
String[] aliases = new String[]{alias};
@ -68,7 +78,7 @@ public class AliasTest extends TestBase {
.indices(indices)
.aliases(aliases);
indicesAliasesRequest.addAliasAction(aliasAction);
client.admin().indices().aliases(indicesAliasesRequest).actionGet();
client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
GetAliasesRequest getAliasesRequest = new GetAliasesRequest();
getAliasesRequest.aliases(alias);
@ -90,5 +100,4 @@ public class AliasTest extends TestBase {
assertEquals("test20160101", it.next());
logger.info("success: result={}", result);
}
}

50
elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java
View file

@ -1,50 +0,0 @@
package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@Ignore
public class ClusterBlockTest extends TestBase {
private static final Logger logger = LogManager.getLogger("test");
@Before
public void startNodes() {
try {
setClusterName("test-cluster-" + System.getProperty("user.name"));
startNode("1");
// do not wait for green health state
logger.info("ready");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
}
@Override
protected Settings getNodeSettings() {
return Settings.builder()
.put(super.getNodeSettings())
.put("discovery.zen.minimum_master_nodes", 2) // block until we have two nodes
.build();
}
@Test(expected = ClusterBlockException.class)
public void testClusterBlock() throws Exception {
Client client = client("1");
XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("field1", "value1").endObject();
IndexRequestBuilder irb = client.prepareIndex("test", "test", "1").setSource(builder);
BulkRequestBuilder brb = client.prepareBulk();
brb.add(irb);
brb.execute().actionGet();
}
}

8
elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java
View file

@ -1,18 +1,18 @@
package org.xbib.elx.common.test;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.MockExtendedClient;
import org.xbib.elx.common.MockExtendedClientProvider;
import java.io.IOException;
import static org.junit.Assert.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotNull;
public class MockExtendedClientProviderTest {
class MockExtendedClientProviderTest {
@Test
public void testMockExtendedProvider() throws IOException {
void testMockExtendedProvider() throws IOException {
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
assertNotNull(client);
}

4
elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java
View file

@ -7,9 +7,9 @@ import org.elasticsearch.plugins.Plugin;
import java.util.List;
public class MockNode extends Node {
class MockNode extends Node {
public MockNode(Settings settings, List<Class<? extends Plugin>> classpathPlugins) {
MockNode(Settings settings, List<Class<? extends Plugin>> classpathPlugins) {
super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
}
}

10
elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java
View file

@ -2,20 +2,22 @@ package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.util.Collections;
import java.util.Enumeration;
public class NetworkTest {
// walk over all found interfaces (this is slow - multicast/pings are performed)
@Disabled
class NetworkTest {
private static final Logger logger = LogManager.getLogger(NetworkTest.class);
@Test
public void testNetwork() throws Exception {
// walk over all found interfaces (this is slow - multicast/pings are performed)
void testNetwork() throws Exception {
Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
for (NetworkInterface netint : Collections.list(nets)) {
System.out.println("checking network interface = " + netint.getName());

49
elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java
View file

@ -1,25 +1,36 @@
package org.xbib.elx.common.test;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.bulk.BulkAction;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class SearchTest extends TestBase {
@ExtendWith(TestExtension.class)
class SearchTest {
private final TestExtension.Helper helper;
SearchTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSearch() throws Exception {
Client client = client("1");
void testSearch() throws Exception {
ElasticsearchClient client = helper.client("1");
BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE);
for (int i = 0; i < 1000; i++) {
IndexRequest indexRequest = new IndexRequest("pages", "row")
@ -39,18 +50,20 @@ public class SearchTest extends TestBase {
.endObject());
builder.add(indexRequest);
}
client.bulk(builder.request()).actionGet();
client.admin().indices().refresh(new RefreshRequest()).actionGet();
for (int i = 0; i < 100; i++) {
client.execute(BulkAction.INSTANCE, builder.request()).actionGet();
client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet();
for (int i = 0; i < 1; i++) {
QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234);
SearchRequestBuilder requestBuilder = client.prepareSearch()
.setIndices("pages")
.setTypes("row")
.setQuery(queryStringBuilder)
.addSort("rowcount", SortOrder.DESC)
.setFrom(i * 10).setSize(10);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
SearchSourceBuilder searchSource = new SearchSourceBuilder();
searchSource.query(queryStringBuilder);
searchSource.sort("rowcount", SortOrder.DESC);
searchSource.from(i * 10);
searchSource.size(10);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("pages");
searchRequest.types("row");
searchRequest.source(searchSource);
SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertTrue(searchResponse.getHits().getTotalHits() > 0);
}
}

30
elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java
View file

@ -1,7 +1,5 @@
package org.xbib.elx.common.test;
import static org.junit.Assert.assertEquals;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
@ -10,7 +8,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.common.settings.Settings;
@ -18,16 +15,26 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
public class SimpleTest extends TestBase {
import static org.junit.jupiter.api.Assertions.assertEquals;
@ExtendWith(TestExtension.class)
class SimpleTest {
private final TestExtension.Helper helper;
SimpleTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void test() throws Exception {
void testSimple() throws Exception {
try {
DeleteIndexRequest deleteIndexRequest =
new DeleteIndexRequest().indices("test");
client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
helper.client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
} catch (IndexNotFoundException e) {
// ignore if index not found
}
@ -38,24 +45,23 @@ public class SimpleTest extends TestBase {
.build();
CreateIndexRequest createIndexRequest = new CreateIndexRequest();
createIndexRequest.index("test").settings(indexSettings);
client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
helper.client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
IndexRequest indexRequest = new IndexRequest();
indexRequest.index("test").type("test").id("1")
.source(XContentFactory.jsonBuilder().startObject().field("field",
"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject());
client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
helper.client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
RefreshRequest refreshRequest = new RefreshRequest();
refreshRequest.indices("test");
client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
helper.client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchQuery("field",
"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8"));
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test").types("test");
searchRequest.source(builder);
String doc = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet()
String doc = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet()
.getHits().getAt(0).getSourceAsString();
assertEquals(doc,
"{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}");
}

207
elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java
View file

@ -1,207 +0,0 @@
package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
setClusterName("test-cluster-" + System.getProperty("user.name"));
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.all();
ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("host", host)
.put("port", port)
.put("cluster.name", cluster)
.put("path.home", getHome())
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected void setClusterName(String cluster) {
this.cluster = cluster;
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
logger.info("clients={}", clients);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

224
elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java Normal file
View file

@ -0,0 +1,224 @@
package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress();
String host = address.address().getHostName();
int port = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

53
elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java
View file

@ -6,43 +6,52 @@ import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import java.io.IOException;
public class WildcardTest extends TestBase {
@ExtendWith(TestExtension.class)
class WildcardTest {
@Test
public void testWildcard() throws Exception {
index(client("1"), "1", "010");
index(client("1"), "2", "0*0");
// exact
validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1);
validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1);
// pattern
validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2?
validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2?
validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2?
validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0);
validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1);
validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1);
validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1?
private final TestExtension.Helper helper;
WildcardTest(TestExtension.Helper helper) {
this.helper = helper;
}
private void index(Client client, String id, String fieldValue) throws IOException {
@Test
void testWildcard() throws Exception {
ElasticsearchClient client = helper.client("1");
index(client, "1", "010");
index(client, "2", "0*0");
// exact
validateCount(client, QueryBuilders.queryStringQuery("010").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1);
// pattern
validateCount(client, QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2?
validateCount(client, QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2?
validateCount(client, QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2?
validateCount(client, QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0);
validateCount(client, QueryBuilders.queryStringQuery("*10").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
validateCount(client, QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1?
}
private void index(ElasticsearchClient client, String id, String fieldValue) throws IOException {
client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id)
.source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject()))
.actionGet();
client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet();
}
private long count(Client client, QueryBuilder queryBuilder) {
private long count(ElasticsearchClient client, QueryBuilder queryBuilder) {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(queryBuilder);
SearchRequest searchRequest = new SearchRequest();
@ -52,7 +61,7 @@ public class WildcardTest extends TestBase {
return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
}
private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) {
private void validateCount(ElasticsearchClient client, QueryBuilder queryBuilder, long expectedHits) {
final long actualHits = count(client, queryBuilder);
if (actualHits != expectedHits) {
throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits);

21
elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java
View file

@ -15,7 +15,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.ClientBuilder;
import java.io.IOException;
import java.util.ArrayList;
@ -59,14 +61,26 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
if (settings == null) {
return null;
}
this.url = settings.get("url");
if (settings.hasValue("url")) {
this.url = settings.get("url");
} else if (settings.hasValue("host")) {
this.url = URL.http()
.host(settings.get("host")).port(settings.getAsInt("port", 9200))
.build()
.toExternalForm();
}
ServiceLoader<HttpAction> httpActionServiceLoader = ServiceLoader.load(HttpAction.class, classLoader);
for (HttpAction<? extends ActionRequest, ? extends ActionResponse> httpAction : httpActionServiceLoader) {
httpAction.setSettings(settings);
actionMap.put(httpAction.getActionInstance(), httpAction);
}
this.nettyHttpClient = Client.builder().enableDebug().build();
logger.info("extended HTTP client initialized with {} actions", actionMap.size());
ClientBuilder clientBuilder = Client.builder();
if (settings.hasValue("debug")) {
clientBuilder.enableDebug();
}
this.nettyHttpClient = clientBuilder.build();
logger.info("extended HTTP client initialized, settings = {}, url = {}, {} actions",
settings, url, actionMap.size());
return this;
}
@ -102,7 +116,6 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response>
execute(Action<Request, Response, RequestBuilder> action, Request request) {
PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
logger.info("plain action future = " + actionFuture);
execute(action, request, actionFuture);
return actionFuture;
}

56
elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java
View file

@ -12,10 +12,11 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
@ -26,34 +27,31 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class ClientTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
private static final Long ACTIONS = 25000L;
private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
@Before
public void startNodes() {
try {
super.startNodes();
startNode("2");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSingleDoc() throws Exception {
void testSingleDoc() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
@ -75,9 +73,10 @@ public class ClientTest extends TestBase {
}
@Test
public void testNewIndex() throws Exception {
void testNewIndex() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
@ -85,12 +84,13 @@ public class ClientTest extends TestBase {
}
@Test
public void testMapping() throws Exception {
void testMapping() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
XContentBuilder builder = jsonBuilder()
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
@ -110,17 +110,18 @@ public class ClientTest extends TestBase {
}
@Test
public void testRandomDocs() throws Exception {
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -144,13 +145,14 @@ public class ClientTest extends TestBase {
}
@Test
public void testThreadedRandomDocs() throws Exception {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
@ -168,7 +170,7 @@ public class ClientTest extends TestBase {
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});

28
elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java
View file

@ -5,8 +5,9 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
@ -15,12 +16,13 @@ import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class DuplicateIDTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
@ -28,17 +30,25 @@ public class DuplicateIDTest extends TestBase {
private static final Long ACTIONS = 12345L;
private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testDuplicateDocIDs() throws Exception {
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);

88
elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java Normal file
View file

@ -0,0 +1,88 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Disabled
@ExtendWith(TestExtension.class)
class IndexPruneTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexPruneTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testPrune() throws IOException {
final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1"))
.put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test1", settings);
client.shiftIndex("test", "test1", Collections.emptyList());
client.newIndex("test2", settings);
client.shiftIndex("test", "test2", Collections.emptyList());
client.newIndex("test3", settings);
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

30
elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java
View file

@ -6,29 +6,37 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class IndexShiftTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class IndexShiftTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName());
private final TestExtension.Helper helper;
IndexShiftTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testIndexShift() throws Exception {
void testIndexShift() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build();
try {
@ -38,7 +46,8 @@ public class IndexShiftTest extends TestBase {
.build();
client.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -66,7 +75,8 @@ public class IndexShiftTest extends TestBase {
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);

151
elx-http/src/test/java/org/xbib/elx/http/test/ReplicaTest.java
View file

@ -1,151 +0,0 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexingStats;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@Ignore
public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName());
@Test
public void testReplicaLevel() throws Exception {
// we need nodes for replica levels
startNode("2");
startNode("3");
startNode("4");
Settings settingsTest1 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 3)
.build();
Settings settingsTest2 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 1)
.build();
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.build();
try {
client.newIndex("test1", settingsTest1, new HashMap<>())
.newIndex("test2", settingsTest2, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 1234; i++) {
client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
logger.info("refreshing");
client.refreshIndex("test1");
client.refreshIndex("test2");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setIndices("test1", "test2")
.setQuery(matchAllQuery());
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE)
.all();
IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal();
IndexingStats indexingStats = commonStats.getIndexing();
IndexingStats.Stats stats = indexingStats.getTotal();
logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
for (Map.Entry<Integer, IndexShardStats> me : indexStats.getIndexShards().entrySet()) {
IndexShardStats indexShardStats = me.getValue();
CommonStats commonShardStats = indexShardStats.getTotal();
logger.info("shard {} count = {}", me.getKey(),
commonShardStats.getIndexing().getTotal().getIndexCount());
}
}
try {
client.deleteIndex("test1")
.deleteIndex("test2");
} catch (Exception e) {
logger.error("delete index failed, ignored. Reason:", e);
}
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
@Test
public void testUpdateReplicaLevel() throws Exception {
long numberOfShards = 2;
int replicaLevel = 3;
// we need 3 nodes for replica level 3
startNode("2");
startNode("3");
Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", 0)
.build();
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.build();
try {
client.newIndex("replicatest", settings, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 12345; i++) {
client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS);
assertEquals(replicaLevel, client.getReplicaLevel("replicatest"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

33
elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java
View file

@ -4,8 +4,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
@ -13,37 +14,39 @@ import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@Ignore
public class SmokeTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class SmokeTest {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName());
private final TestExtension.Helper helper;
SmokeTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void smokeTest() throws Exception {
void smokeTest() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
assertEquals(getClusterName(), client.getClusterName());
assertEquals(helper.getClusterName(), client.getClusterName());
client.checkMapping("test");
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.flush();
client.waitForRecovery("test", 10L, TimeUnit.SECONDS);
client.delete("test", "1");
client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel());
@ -51,10 +54,8 @@ public class SmokeTest extends TestBase {
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());

198
elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java
View file

@ -1,198 +0,0 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private AtomicInteger counter = new AtomicInteger();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
this.cluster = "test-helper-cluster-" + counter.incrementAndGet();
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("host", host)
.put("port", port)
.put("cluster.name", cluster)
.put("path.home", getHome())
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("discovery.zen.minimum_master_nodes", "1")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

236
elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java Normal file
View file

@ -0,0 +1,236 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getHttp().getAddress().publishAddress();
helper.httpHost = address.address().getHostName();
helper.httpPort = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
String httpHost;
int httpPort;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
Settings getHttpSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.put("host", httpHost)
.put("port", httpPort)
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

61
elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java
View file

@ -1,10 +1,5 @@
package org.xbib.elx.node.test;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
@ -17,9 +12,10 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
@ -30,27 +26,28 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class ClientTest extends TestBase {
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 25000L;
private static final Long ACTIONS = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
@Before
public void startNodes() {
try {
super.startNodes();
startNode("2");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSingleDoc() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testSingleDoc() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
@ -73,8 +70,8 @@ public class ClientTest extends TestBase {
}
@Test
public void testNewIndex() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testNewIndex() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
@ -83,12 +80,12 @@ public class ClientTest extends TestBase {
}
@Test
public void testMapping() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testMapping() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
XContentBuilder builder = jsonBuilder()
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
@ -108,9 +105,9 @@ public class ClientTest extends TestBase {
}
@Test
public void testRandomDocs() throws Exception {
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
@ -118,7 +115,8 @@ public class ClientTest extends TestBase {
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -140,12 +138,12 @@ public class ClientTest extends TestBase {
}
@Test
public void testThreadedRandomDocs() throws Exception {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
@ -164,7 +162,8 @@ public class ClientTest extends TestBase {
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});

30
elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java
View file

@ -7,7 +7,8 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
@ -15,28 +16,37 @@ import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class DuplicateIDTest extends TestBase {
@ExtendWith(TestExtension.class)
class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 12345L;
private static final Long ACTIONS = 5L;
private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testDuplicateDocIDs() throws Exception {
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -47,7 +57,7 @@ public class DuplicateIDTest extends TestBase {
searchRequest.indices("test");
searchRequest.types("test");
searchRequest.source(builder);
long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {

25
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java
View file

@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
@ -19,17 +20,24 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class IndexPruneTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexPruneTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexPruneTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
@ -45,15 +53,12 @@ public class IndexPruneTest extends TestBase {
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();

28
elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java
View file

@ -6,7 +6,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
@ -16,16 +17,23 @@ import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class IndexShiftTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexShiftTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexShiftTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testIndexShift() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testIndexShift() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
@ -35,14 +43,13 @@ public class IndexShiftTest extends TestBase {
.build();
client.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -63,7 +70,8 @@ public class IndexShiftTest extends TestBase {
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);

151
elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java
View file

@ -1,151 +0,0 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexingStats;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@Ignore
public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName());
@Test
public void testReplicaLevel() throws Exception {
// we need nodes for replica levels
startNode("2");
startNode("3");
startNode("4");
Settings settingsTest1 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 3)
.build();
Settings settingsTest2 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 1)
.build();
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
client.newIndex("test1", settingsTest1, new HashMap<>())
.newIndex("test2", settingsTest2, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 1234; i++) {
client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
logger.info("refreshing");
client.refreshIndex("test1");
client.refreshIndex("test2");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setIndices("test1", "test2")
.setQuery(matchAllQuery());
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE)
.all();
IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal();
IndexingStats indexingStats = commonStats.getIndexing();
IndexingStats.Stats stats = indexingStats.getTotal();
logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
for (Map.Entry<Integer, IndexShardStats> me : indexStats.getIndexShards().entrySet()) {
IndexShardStats indexShardStats = me.getValue();
CommonStats commonShardStats = indexShardStats.getTotal();
logger.info("shard {} count = {}", me.getKey(),
commonShardStats.getIndexing().getTotal().getIndexCount());
}
}
try {
client.deleteIndex("test1")
.deleteIndex("test2");
} catch (Exception e) {
logger.error("delete index failed, ignored. Reason:", e);
}
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
@Test
public void testUpdateReplicaLevel() throws Exception {
long numberOfShards = 2;
int replicaLevel = 3;
// we need 3 nodes for replica level 3
startNode("2");
startNode("3");
Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", 0)
.build();
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
client.newIndex("replicatest", settings, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 12345; i++) {
client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS);
assertEquals(replicaLevel, client.getReplicaLevel("replicatest"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

27
elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java
View file

@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.node.ExtendedNodeClient;
@ -12,31 +13,35 @@ import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
public class SmokeTest extends TestBase {
@ExtendWith(TestExtension.class)
class SmokeTest {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
private final TestExtension.Helper helper;
SmokeTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void smokeTest() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void smokeTest() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
assertEquals(getClusterName(), client.getClusterName());
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.delete("test", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test");
client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel());
@ -46,10 +51,8 @@ public class SmokeTest extends TestBase {
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());

204
elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java
View file

@ -1,204 +0,0 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
this.cluster = "test-cluster-" + System.getProperty("user.name");
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.all();
ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("host", host)
.put("port", port)
.put("cluster.name", cluster)
.put("path.home", getHome())
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("discovery.zen.minimum_master_nodes", "1")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

236
elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java Normal file
View file

@ -0,0 +1,236 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress();
helper.host = address.address().getHostName();
helper.port = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
String host;
int port;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
Settings getTransportSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("path.home", getHome())
.put("host", host)
.put("port", port)
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

59
elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java
View file

@ -13,10 +13,11 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -27,36 +28,32 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class ClientTest extends TestBase {
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 25000L;
private static final Long ACTIONS = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
@Before
public void startNodes() {
try {
super.startNodes();
startNode("2");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSingleDoc() throws Exception {
void testSingleDoc() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test");
@ -76,24 +73,24 @@ public class ClientTest extends TestBase {
}
@Test
public void testNewIndex() throws Exception {
void testNewIndex() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
client.newIndex("test");
client.close();
}
@Test
public void testMapping() throws Exception {
void testMapping() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
XContentBuilder builder = jsonBuilder()
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
@ -113,18 +110,19 @@ public class ClientTest extends TestBase {
}
@Test
public void testRandomDocs() throws Exception {
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -152,7 +150,7 @@ public class ClientTest extends TestBase {
}
@Test
public void testThreadedRandomDocs() throws Exception {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
@ -162,7 +160,7 @@ public class ClientTest extends TestBase {
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
@ -177,7 +175,8 @@ public class ClientTest extends TestBase {
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});

31
elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java
View file

@ -8,7 +8,8 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -16,30 +17,38 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class DuplicateIDTest extends TestBase {
@ExtendWith(TestExtension.class)
class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 12345L;
private static final Long ACTIONS = 5L;
private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testDuplicateDocIDs() throws Exception {
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -52,7 +61,7 @@ public class DuplicateIDTest extends TestBase {
searchRequest.types("test");
searchRequest.source(builder);
SearchResponse searchResponse =
client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);

25
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java
View file

@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -19,19 +20,26 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class IndexPruneTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexPruneTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexPruneTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testPrune() throws IOException {
void testPrune() throws IOException {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
@ -46,15 +54,12 @@ public class IndexPruneTest extends TestBase {
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();

40
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java
View file

@ -6,31 +6,36 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class IndexShiftTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexShiftTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexShiftTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testIndexShift() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder(client("1"))
void testIndexShift() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
@ -39,39 +44,35 @@ public class IndexShiftTest extends TestBase {
.build();
client.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = client.getAliases("test1234");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
String resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
@ -82,7 +83,6 @@ public class IndexShiftTest extends TestBase {
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = client.getAliases("test5678");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
@ -90,7 +90,6 @@ public class IndexShiftTest extends TestBase {
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
@ -99,7 +98,6 @@ public class IndexShiftTest extends TestBase {
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {

161
elx-transport/src/test/java/org/xbib/elx/transport/test/ReplicaTest.java
View file

@ -1,161 +0,0 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.shard.IndexingStats;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@Ignore
public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName());
@Test
public void testReplicaLevel() throws Exception {
// we need nodes for replica levels
startNode("2");
startNode("3");
startNode("4");
Settings settingsTest1 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 3)
.build();
Settings settingsTest2 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 1)
.build();
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.build();
try {
client.newIndex("test1", settingsTest1, new HashMap<>())
.newIndex("test2", settingsTest2, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 1234; i++) {
client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
logger.info("refreshing");
client.refreshIndex("test1");
client.refreshIndex("test2");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(0);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test1", "test2");
searchRequest.source(builder);
SearchResponse searchResponse =
client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
indicesStatsRequest.all();
IndicesStatsResponse response =
client.getClient().execute(IndicesStatsAction.INSTANCE, indicesStatsRequest).actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal();
IndexingStats indexingStats = commonStats.getIndexing();
IndexingStats.Stats stats = indexingStats.getTotal();
logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
for (Map.Entry<Integer, IndexShardStats> me : indexStats.getIndexShards().entrySet()) {
IndexShardStats indexShardStats = me.getValue();
CommonStats commonShardStats = indexShardStats.getTotal();
logger.info("shard {} count = {}", me.getKey(),
commonShardStats.getIndexing().getTotal().getIndexCount());
}
}
try {
client.deleteIndex("test1")
.deleteIndex("test2");
} catch (Exception e) {
logger.error("delete index failed, ignored. Reason:", e);
}
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
@Test
public void testUpdateReplicaLevel() throws Exception {
long numberOfShards = 2;
int replicaLevel = 3;
// we need 3 nodes for replica level 3
startNode("2");
startNode("3");
Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", 0)
.build();
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.build();
try {
client.newIndex("replicatest", settings, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 12345; i++) {
client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS);
assertEquals(replicaLevel, client.getReplicaLevel("replicatest"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

25
elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java
View file

@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -12,22 +13,28 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
public class SmokeTest extends TestBase {
@ExtendWith(TestExtension.class)
class SmokeTest {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
@Test
public void smokeTest() throws Exception {
private final TestExtension.Helper helper;
SmokeTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void smokeTest() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
assertEquals(getClusterName(), client.getClusterName());
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.update("test", "1", "{ \"name\" : \"Another name\"}");
@ -36,7 +43,6 @@ public class SmokeTest extends TestBase {
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test");
client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel());
@ -47,7 +53,6 @@ public class SmokeTest extends TestBase {
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());

204
elx-transport/src/test/java/org/xbib/elx/transport/test/TestBase.java
View file

@ -1,204 +0,0 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
this.cluster = "test-cluster-" + System.getProperty("user.name");
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.all();
ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("path.home", getHome())
.put("host", host)
.put("port", port)
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("discovery.zen.minimum_master_nodes", "1")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

236
elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java Normal file
View file

@ -0,0 +1,236 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress();
helper.host = address.address().getHostName();
helper.port = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
String host;
int port;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
Settings getTransportSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("path.home", getHome())
.put("host", host)
.put("port", port)
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

10
gradle.properties
View file

@ -1,17 +1,17 @@
group = org.xbib
name = elx
version = 6.3.2.1
version = 6.3.2.2
profile = default
release = 0
elasticsearch-server.version = 6.3.2.2
elasticsearch-server.version = 6.3.2.3
log4j.version = 2.11.1
xbib-metrics.version = 1.2.0
xbib-netty-http.version = 4.1.33.0
xbib-netty-http.version = 4.1.35.0
# test
junit.version = 4.12
wagon.version = 3.0.0
junit.version = 5.4.2
wagon.version = 3.3.2
asciidoclet.version = 1.6.0.0
org.gradle.warning.mode = all

BIN
gradle/wrapper/gradle-wrapper.jar vendored
View file

Binary file not shown.

3
gradle/wrapper/gradle-wrapper.properties vendored
View file

@ -1,6 +1,5 @@
#Fri Feb 15 10:39:20 CET 2019
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip

18
gradlew vendored
View file

@ -1,5 +1,21 @@
#!/usr/bin/env sh
#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
##
## Gradle start up script for UN*X
@ -28,7 +44,7 @@ APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m"'
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"

18
gradlew.bat vendored
View file

@ -1,3 +1,19 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m"
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome