joerg
/
elx
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update to Junit 5.4.2, ES 6.3.2.3, gradle 5.3, fix tests

Browse Source
main 6.3.2.2
Jörg Prante 5 years ago
parent 2b3ac518b5
commit d4e40145be
47 changed files with 1584 additions and 1714 deletions
Show Stats Download Patch File Download Diff File

84
build.gradle
Unescape Escape View File

@ -16,12 +16,12 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s
System.getProperty("java.vm.version"),
System.getProperty("java.vm.vendor"),
System.getProperty("java.vm.name"),
gradle.gradleVersion,
GroovySystem.getVersion(),
gradle.gradleVersion,
GroovySystem.getVersion(),
JavaVersion.current()
if (JavaVersion.current() < JavaVersion.VERSION_11) {
throw new GradleException("This build must be run with java 11 or higher")
throw new GradleException("This build must be run with java 11")
}
subprojects {
@ -39,9 +39,11 @@ subprojects {
}
dependencies {
testCompile "junit:junit:${project.property('junit.version')}"
testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}"
testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}"
testImplementation "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}"
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}"
testImplementation "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}"
testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}"
testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}"
asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}"
wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}"
}
@ -63,21 +65,35 @@ subprojects {
}
test {
jvmArgs =[
'--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED',
'--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED',
'--add-opens=java.base/java.nio=ALL-UNNAMED'
]
enabled = true
useJUnitPlatform()
// we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery
doFirst {
jvmArgs = [
'--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED',
'--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED',
'--add-opens=java.base/java.nio=ALL-UNNAMED'
]
}
systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager'
systemProperty 'path.home', "${project.buildDir}"
systemProperty 'jna.debug_load', 'true'
testLogging {
showStandardStreams = true
exceptionFormat = 'full'
events 'PASSED', 'FAILED', 'SKIPPED'
}
afterSuite { desc, result ->
if (!desc.parent) {
println "\nTest result: ${result.resultType}"
println "Test summary: ${result.testCount} tests, " +
"${result.successfulTestCount} succeeded, " +
"${result.failedTestCount} failed, " +
"${result.skippedTestCount} skipped"
}
}
}
clean {
delete "data"
delete "logs"
delete "out"
}
@ -94,18 +110,18 @@ subprojects {
}*/
task javadocJar(type: Jar, dependsOn: javadoc) {
classifier 'javadoc'
archiveClassifier.set('javadoc')
}
task sourcesJar(type: Jar, dependsOn: classes) {
from sourceSets.main.allSource
classifier 'sources'
archiveClassifier.set('sources')
}
artifacts {
archives javadocJar, sourcesJar
}
if (project.hasProperty('signing.keyId')) {
signing {
sign configurations.archives
@ -164,35 +180,3 @@ subprojects {
}
}
}
/*asciidoctor {
attributes toc: 'left',
doctype: 'book',
icons: 'font',
encoding: 'utf-8',
sectlink: true,
sectanchors: true,
linkattrs: true,
imagesdir: 'img',
'source-highlighter': 'coderay'
}*/
/*
task aggregatedJavadoc(type: Javadoc) {
group = 'aggregation'
description = 'Generates aggregated Javadoc API documentation.'
title = "$description $version API"
destinationDir = file("$buildDir/docs/javadoc")
def sourceProjects = subprojects.findAll {
it.plugins.hasPlugin('java') || it.plugins.hasPlugin('groovy')
}
source sourceProjects.collect {
it.sourceSets.main.allJava
}
classpath = files(sourceProjects.collect {
it.sourceSets.main.runtimeClasspath
})
//options.overview = 'gradle/api/overview.html'
options.showFromProtected()
}
*/

4
elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java
Unescape Escape View File

@ -398,9 +398,9 @@ public interface ExtendedClient extends Flushable, Closeable {
String resolveMostRecentIndex(String alias);
/**
* Get all index filters.
* Get all index aliases.
* @param index the index
* @return map of index filters
* @return map of index aliases
*/
Map<String, String> getAliases(String index);

25
elx-api/src/main/java/org/xbib/elx/api/ReadClient.java
Unescape Escape View File

@ -0,0 +1,25 @@
package org.xbib.elx.api;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
public interface ReadClient {
ActionFuture<GetResponse> get(GetRequest getRequest);
void get(GetRequest request, ActionListener<GetResponse> listener);
ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
ActionFuture<SearchResponse> search(SearchRequest request);
void search(SearchRequest request, ActionListener<SearchResponse> listener);
}

6
elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java
Unescape Escape View File

@ -0,0 +1,6 @@
package org.xbib.elx.api;
public interface ReadClientProvider<C extends ReadClient> {
C getReadClient();
}

3
elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java
Unescape Escape View File

@ -454,7 +454,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
.indices(index)
.waitForActiveShards(shards).timeout(timeout);
.waitForActiveShards(shards)
.timeout(timeout);
ClusterHealthResponse healthResponse =
client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {

23
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java
Unescape Escape View File

@ -8,7 +8,6 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
@ -83,12 +82,12 @@ public class DefaultBulkController implements BulkController {
maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest);
}
this.bulkListener = new BulkListener();
DefaultBulkProcessor.Builder builder = DefaultBulkProcessor.builder((Client) client.getClient(), bulkListener)
this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener)
.setBulkActions(maxActionsPerRequest)
.setConcurrentRequests(maxConcurrentRequests)
.setFlushInterval(flushIngestInterval)
.setBulkSize(maxVolumePerRequest);
this.bulkProcessor = builder.build();
.setBulkSize(maxVolumePerRequest)
.build();
this.active.set(true);
}
@ -115,9 +114,7 @@ public class DefaultBulkController implements BulkController {
@Override
public void index(IndexRequest indexRequest) {
if (!active.get()) {
throw new IllegalStateException("inactive");
}
ensureActiveAndBulk();
try {
if (bulkMetric != null) {
bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
@ -226,6 +223,18 @@ public class DefaultBulkController implements BulkController {
}
}
private void ensureActiveAndBulk() {
if (!active.get()) {
throw new IllegalStateException("inactive");
}
if (bulkProcessor == null) {
throw new UnsupportedOperationException("bulk processor not present");
}
if (bulkListener == null) {
throw new UnsupportedOperationException("bulk listener not present");
}
}
private class BulkListener implements DefaultBulkProcessor.Listener {
private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener");

42
elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java
Unescape Escape View File

@ -8,7 +8,8 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
@ -16,6 +17,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.util.concurrent.FutureUtils;
import org.xbib.elx.api.BulkProcessor;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
@ -47,7 +49,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
private volatile boolean closed;
private DefaultBulkProcessor(Client client, Listener listener, String name, int concurrentRequests,
private DefaultBulkProcessor(ElasticsearchClient client, Listener listener, String name, int concurrentRequests,
int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) {
this.executionIdGen = new AtomicLong();
this.closed = false;
@ -59,7 +61,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
new AsyncBulkRequestHandler(client, listener, concurrentRequests);
if (flushInterval != null) {
this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1,
EsExecutors.daemonThreadFactory(client.settings(),
EsExecutors.daemonThreadFactory(Settings.EMPTY,
name != null ? "[" + name + "]" : "" + "bulk_processor"));
this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
@ -71,10 +73,9 @@ public class DefaultBulkProcessor implements BulkProcessor {
}
}
public static Builder builder(Client client, Listener listener) {
if (client == null) {
throw new NullPointerException("The client you specified while building a BulkProcessor is null");
}
public static Builder builder(ElasticsearchClient client, Listener listener) {
Objects.requireNonNull(client, "The client you specified while building a BulkProcessor is null");
Objects.requireNonNull(listener, "A listener for the BulkProcessor is required but null");
return new Builder(client, listener);
}
@ -87,6 +88,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
*/
@Override
public synchronized boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException {
Objects.requireNonNull(unit, "A time unit is required for awaitFlush() but null");
if (closed) {
return true;
}
@ -95,7 +97,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
execute();
}
// wait for all bulk responses
return this.bulkRequestHandler.close(timeout, unit);
return bulkRequestHandler.close(timeout, unit);
}
/**
@ -115,18 +117,19 @@ public class DefaultBulkProcessor implements BulkProcessor {
*/
@Override
public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException {
Objects.requireNonNull(unit, "A time unit is required for awaitCLose() but null");
if (closed) {
return true;
}
closed = true;
if (this.scheduledFuture != null) {
FutureUtils.cancel(this.scheduledFuture);
if (scheduledFuture != null) {
FutureUtils.cancel(scheduledFuture);
this.scheduler.shutdown();
}
if (bulkRequest.numberOfActions() > 0) {
execute();
}
return this.bulkRequestHandler.close(timeout, unit);
return bulkRequestHandler.close(timeout, unit);
}
/**
@ -215,8 +218,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
private boolean isOverTheLimit() {
return bulkActions != -1 &&
bulkRequest.numberOfActions() >= bulkActions ||
bulkSize != -1 &&
bulkRequest.estimatedSizeInBytes() >= bulkSize;
bulkSize != -1 && bulkRequest.estimatedSizeInBytes() >= bulkSize;
}
/**
@ -224,7 +226,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
*/
public static class Builder {
private final Client client;
private final ElasticsearchClient client;
private final Listener listener;
@ -245,7 +247,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
* @param client the client
* @param listener the listener
*/
Builder(Client client, Listener listener) {
Builder(ElasticsearchClient client, Listener listener) {
this.client = client;
this.listener = listener;
}
@ -339,11 +341,12 @@ public class DefaultBulkProcessor implements BulkProcessor {
private static class SyncBulkRequestHandler implements BulkRequestHandler {
private final Client client;
private final ElasticsearchClient client;
private final DefaultBulkProcessor.Listener listener;
SyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener) {
SyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener) {
Objects.requireNonNull(listener, "A listener is required for SyncBulkRequestHandler but null");
this.client = client;
this.listener = listener;
}
@ -371,7 +374,7 @@ public class DefaultBulkProcessor implements BulkProcessor {
private static class AsyncBulkRequestHandler implements BulkRequestHandler {
private final Client client;
private final ElasticsearchClient client;
private final DefaultBulkProcessor.Listener listener;
@ -379,7 +382,8 @@ public class DefaultBulkProcessor implements BulkProcessor {
private final int concurrentRequests;
private AsyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener, int concurrentRequests) {
private AsyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener, int concurrentRequests) {
Objects.requireNonNull(listener, "A listener is required for AsyncBulkRequestHandler but null");
this.client = client;
this.listener = listener;
this.concurrentRequests = concurrentRequests;

49
elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java
Unescape Escape View File

@ -1,19 +1,19 @@
package org.xbib.elx.common.test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.Strings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import java.util.Collections;
import java.util.Iterator;
@ -22,44 +22,54 @@ import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class AliasTest extends TestBase {
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class AliasTest {
private static final Logger logger = LogManager.getLogger(AliasTest.class.getName());
private final TestExtension.Helper helper;
AliasTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testAlias() {
Client client = client("1");
void testAlias() {
ElasticsearchClient client = helper.client("1");
CreateIndexRequest indexRequest = new CreateIndexRequest("test");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
// put alias
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
String[] indices = new String[]{"test"};
String[] aliases = new String[]{"test_alias"};
String[] indices = new String[] { "test" };
String[] aliases = new String[] { "test_alias" };
IndicesAliasesRequest.AliasActions aliasAction =
new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD)
.indices(indices)
.aliases(aliases);
indicesAliasesRequest.addAliasAction(aliasAction);
client.admin().indices().aliases(indicesAliasesRequest).actionGet();
client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
// get alias
GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY);
long t0 = System.nanoTime();
GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(getAliasesRequest).actionGet();
GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
long t1 = (System.nanoTime() - t0) / 1000000;
logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1);
assertTrue(t1 >= 0);
}
@Test
public void testMostRecentIndex() {
Client client = client("1");
void testMostRecentIndex() {
ElasticsearchClient client = helper.client("1");
String alias = "test";
CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
indexRequest = new CreateIndexRequest("test20160102");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
indexRequest = new CreateIndexRequest("test20160103");
client.admin().indices().create(indexRequest).actionGet();
client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet();
IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
String[] indices = new String[]{"test20160101", "test20160102", "test20160103"};
String[] aliases = new String[]{alias};
@ -68,7 +78,7 @@ public class AliasTest extends TestBase {
.indices(indices)
.aliases(aliases);
indicesAliasesRequest.addAliasAction(aliasAction);
client.admin().indices().aliases(indicesAliasesRequest).actionGet();
client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
GetAliasesRequest getAliasesRequest = new GetAliasesRequest();
getAliasesRequest.aliases(alias);
@ -90,5 +100,4 @@ public class AliasTest extends TestBase {
assertEquals("test20160101", it.next());
logger.info("success: result={}", result);
}
}

50
elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java
Unescape Escape View File

@ -1,50 +0,0 @@
package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
@Ignore
public class ClusterBlockTest extends TestBase {
private static final Logger logger = LogManager.getLogger("test");
@Before
public void startNodes() {
try {
setClusterName("test-cluster-" + System.getProperty("user.name"));
startNode("1");
// do not wait for green health state
logger.info("ready");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
}
@Override
protected Settings getNodeSettings() {
return Settings.builder()
.put(super.getNodeSettings())
.put("discovery.zen.minimum_master_nodes", 2) // block until we have two nodes
.build();
}
@Test(expected = ClusterBlockException.class)
public void testClusterBlock() throws Exception {
Client client = client("1");
XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("field1", "value1").endObject();
IndexRequestBuilder irb = client.prepareIndex("test", "test", "1").setSource(builder);
BulkRequestBuilder brb = client.prepareBulk();
brb.add(irb);
brb.execute().actionGet();
}
}

8
elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java
Unescape Escape View File

@ -1,18 +1,18 @@
package org.xbib.elx.common.test;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.MockExtendedClient;
import org.xbib.elx.common.MockExtendedClientProvider;
import java.io.IOException;
import static org.junit.Assert.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotNull;
public class MockExtendedClientProviderTest {
class MockExtendedClientProviderTest {
@Test
public void testMockExtendedProvider() throws IOException {
void testMockExtendedProvider() throws IOException {
MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
assertNotNull(client);
}

4
elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java
Unescape Escape View File

@ -7,9 +7,9 @@ import org.elasticsearch.plugins.Plugin;
import java.util.List;
public class MockNode extends Node {
class MockNode extends Node {
public MockNode(Settings settings, List<Class<? extends Plugin>> classpathPlugins) {
MockNode(Settings settings, List<Class<? extends Plugin>> classpathPlugins) {
super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
}
}

10
elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java
Unescape Escape View File

@ -2,20 +2,22 @@ package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.util.Collections;
import java.util.Enumeration;
public class NetworkTest {
// walk over all found interfaces (this is slow - multicast/pings are performed)
@Disabled
class NetworkTest {
private static final Logger logger = LogManager.getLogger(NetworkTest.class);
@Test
public void testNetwork() throws Exception {
// walk over all found interfaces (this is slow - multicast/pings are performed)
void testNetwork() throws Exception {
Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
for (NetworkInterface netint : Collections.list(nets)) {
System.out.println("checking network interface = " + netint.getName());

49
elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java
Unescape Escape View File

@ -1,25 +1,36 @@
package org.xbib.elx.common.test;
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.bulk.BulkAction;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class SearchTest extends TestBase {
@ExtendWith(TestExtension.class)
class SearchTest {
private final TestExtension.Helper helper;
SearchTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSearch() throws Exception {
Client client = client("1");
void testSearch() throws Exception {
ElasticsearchClient client = helper.client("1");
BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE);
for (int i = 0; i < 1000; i++) {
IndexRequest indexRequest = new IndexRequest("pages", "row")
@ -39,18 +50,20 @@ public class SearchTest extends TestBase {
.endObject());
builder.add(indexRequest);
}
client.bulk(builder.request()).actionGet();
client.admin().indices().refresh(new RefreshRequest()).actionGet();
for (int i = 0; i < 100; i++) {
client.execute(BulkAction.INSTANCE, builder.request()).actionGet();
client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet();
for (int i = 0; i < 1; i++) {
QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234);
SearchRequestBuilder requestBuilder = client.prepareSearch()
.setIndices("pages")
.setTypes("row")
.setQuery(queryStringBuilder)
.addSort("rowcount", SortOrder.DESC)
.setFrom(i * 10).setSize(10);
SearchResponse searchResponse = requestBuilder.execute().actionGet();
SearchSourceBuilder searchSource = new SearchSourceBuilder();
searchSource.query(queryStringBuilder);
searchSource.sort("rowcount", SortOrder.DESC);
searchSource.from(i * 10);
searchSource.size(10);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("pages");
searchRequest.types("row");
searchRequest.source(searchSource);
SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
assertTrue(searchResponse.getHits().getTotalHits() > 0);
}
}

30
elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java
Unescape Escape View File

@ -1,7 +1,5 @@
package org.xbib.elx.common.test;
import static org.junit.Assert.assertEquals;
import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
@ -10,7 +8,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.common.settings.Settings;
@ -18,16 +15,26 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ExtendWith(TestExtension.class)
class SimpleTest {
public class SimpleTest extends TestBase {
private final TestExtension.Helper helper;
SimpleTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void test() throws Exception {
void testSimple() throws Exception {
try {
DeleteIndexRequest deleteIndexRequest =
new DeleteIndexRequest().indices("test");
client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
helper.client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
} catch (IndexNotFoundException e) {
// ignore if index not found
}
@ -38,24 +45,23 @@ public class SimpleTest extends TestBase {
.build();
CreateIndexRequest createIndexRequest = new CreateIndexRequest();
createIndexRequest.index("test").settings(indexSettings);
client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
helper.client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
IndexRequest indexRequest = new IndexRequest();
indexRequest.index("test").type("test").id("1")
.source(XContentFactory.jsonBuilder().startObject().field("field",
"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject());
client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
helper.client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
RefreshRequest refreshRequest = new RefreshRequest();
refreshRequest.indices("test");
client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
helper.client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchQuery("field",
"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8"));
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test").types("test");
searchRequest.source(builder);
String doc = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet()
String doc = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet()
.getHits().getAt(0).getSourceAsString();
assertEquals(doc,
"{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}");
}

207
elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java
Unescape Escape View File

@ -1,207 +0,0 @@
package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
setClusterName("test-cluster-" + System.getProperty("user.name"));
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.all();
ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("host", host)
.put("port", port)
.put("cluster.name", cluster)
.put("path.home", getHome())
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected void setClusterName(String cluster) {
this.cluster = cluster;
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
logger.info("clients={}", clients);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

224
elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java
Unescape Escape View File

@ -0,0 +1,224 @@
package org.xbib.elx.common.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress();
String host = address.address().getHostName();
int port = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

47
elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java
Unescape Escape View File

@ -6,43 +6,52 @@ import org.elasticsearch.action.index.IndexAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import java.io.IOException;
public class WildcardTest extends TestBase {
@ExtendWith(TestExtension.class)
class WildcardTest {
private final TestExtension.Helper helper;
WildcardTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testWildcard() throws Exception {
index(client("1"), "1", "010");
index(client("1"), "2", "0*0");
void testWildcard() throws Exception {
ElasticsearchClient client = helper.client("1");
index(client, "1", "010");
index(client, "2", "0*0");
// exact
validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1);
validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("010").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1);
// pattern
validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2?
validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2?
validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2?
validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0);
validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1);
validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1);
validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1?
validateCount(client, QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2?
validateCount(client, QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2?
validateCount(client, QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2?
validateCount(client, QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0);
validateCount(client, QueryBuilders.queryStringQuery("*10").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1);
validateCount(client, QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1?
validateCount(client, QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1?
}
private void index(Client client, String id, String fieldValue) throws IOException {
private void index(ElasticsearchClient client, String id, String fieldValue) throws IOException {
client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id)
.source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject()))
.actionGet();
client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet();
}
private long count(Client client, QueryBuilder queryBuilder) {
private long count(ElasticsearchClient client, QueryBuilder queryBuilder) {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(queryBuilder);
SearchRequest searchRequest = new SearchRequest();
@ -52,7 +61,7 @@ public class WildcardTest extends TestBase {
return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
}
private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) {
private void validateCount(ElasticsearchClient client, QueryBuilder queryBuilder, long expectedHits) {
final long actualHits = count(client, queryBuilder);
if (actualHits != expectedHits) {
throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits);

21
elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java
Unescape Escape View File

@ -15,7 +15,9 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.xbib.elx.common.AbstractExtendedClient;
import org.xbib.net.URL;
import org.xbib.netty.http.client.Client;
import org.xbib.netty.http.client.ClientBuilder;
import java.io.IOException;
import java.util.ArrayList;
@ -59,14 +61,26 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
if (settings == null) {
return null;
}
this.url = settings.get("url");
if (settings.hasValue("url")) {
this.url = settings.get("url");
} else if (settings.hasValue("host")) {
this.url = URL.http()
.host(settings.get("host")).port(settings.getAsInt("port", 9200))
.build()
.toExternalForm();
}
ServiceLoader<HttpAction> httpActionServiceLoader = ServiceLoader.load(HttpAction.class, classLoader);
for (HttpAction<? extends ActionRequest, ? extends ActionResponse> httpAction : httpActionServiceLoader) {
httpAction.setSettings(settings);
actionMap.put(httpAction.getActionInstance(), httpAction);
}
this.nettyHttpClient = Client.builder().enableDebug().build();
logger.info("extended HTTP client initialized with {} actions", actionMap.size());
ClientBuilder clientBuilder = Client.builder();
if (settings.hasValue("debug")) {
clientBuilder.enableDebug();
}
this.nettyHttpClient = clientBuilder.build();
logger.info("extended HTTP client initialized, settings = {}, url = {}, {} actions",
settings, url, actionMap.size());
return this;
}
@ -102,7 +116,6 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response>
execute(Action<Request, Response, RequestBuilder> action, Request request) {
PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
logger.info("plain action future = " + actionFuture);
execute(action, request, actionFuture);
return actionFuture;
}

56
elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java
Unescape Escape View File

@ -12,10 +12,11 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
@ -26,34 +27,31 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class ClientTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
private static final Long ACTIONS = 25000L;
private static final Long ACTIONS = 100L;
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
@Before
public void startNodes() {
try {
super.startNodes();
startNode("2");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSingleDoc() throws Exception {
void testSingleDoc() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.build();
@ -75,9 +73,10 @@ public class ClientTest extends TestBase {
}
@Test
public void testNewIndex() throws Exception {
void testNewIndex() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
client.newIndex("test");
@ -85,12 +84,13 @@ public class ClientTest extends TestBase {
}
@Test
public void testMapping() throws Exception {
void testMapping() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
XContentBuilder builder = jsonBuilder()
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
@ -110,17 +110,18 @@ public class ClientTest extends TestBase {
}
@Test
public void testRandomDocs() throws Exception {
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -144,13 +145,14 @@ public class ClientTest extends TestBase {
}
@Test
public void testThreadedRandomDocs() throws Exception {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
@ -168,7 +170,7 @@ public class ClientTest extends TestBase {
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});

28
elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java
Unescape Escape View File

@ -5,8 +5,9 @@ import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.http.ExtendedHttpClient;
@ -15,12 +16,13 @@ import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class DuplicateIDTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
@ -28,17 +30,25 @@ public class DuplicateIDTest extends TestBase {
private static final Long ACTIONS = 12345L;
private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testDuplicateDocIDs() throws Exception {
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);

88
elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java
Unescape Escape View File

@ -0,0 +1,88 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Disabled
@ExtendWith(TestExtension.class)
class IndexPruneTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexPruneTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void testPrune() throws IOException {
final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1"))
.put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build();
try {
Settings settings = Settings.builder()
.put("index.number_of_shards", 1)
.put("index.number_of_replicas", 0)
.build();
client.newIndex("test1", settings);
client.shiftIndex("test", "test1", Collections.emptyList());
client.newIndex("test2", settings);
client.shiftIndex("test", "test2", Collections.emptyList());
client.newIndex("test3", settings);
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
indicesExistsRequest.indices(index);
IndicesExistsResponse indicesExistsResponse =
client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
list.add(indicesExistsResponse.isExists());
}
logger.info(list);
assertFalse(list.get(0));
assertFalse(list.get(1));
assertTrue(list.get(2));
assertTrue(list.get(3));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

30
elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java
Unescape Escape View File

@ -6,29 +6,37 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class IndexShiftTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class IndexShiftTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName());
private final TestExtension.Helper helper;
IndexShiftTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testIndexShift() throws Exception {
void testIndexShift() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.put(helper.getHttpSettings())
.provider(ExtendedHttpClientProvider.class)
.build();
try {
@ -38,7 +46,8 @@ public class IndexShiftTest extends TestBase {
.build();
client.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -66,7 +75,8 @@ public class IndexShiftTest extends TestBase {
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);

151
elx-http/src/test/java/org/xbib/elx/http/test/ReplicaTest.java
Unescape Escape View File

@ -1,151 +0,0 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexingStats;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@Ignore
public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName());
@Test
public void testReplicaLevel() throws Exception {
// we need nodes for replica levels
startNode("2");
startNode("3");
startNode("4");
Settings settingsTest1 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 3)
.build();
Settings settingsTest2 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 1)
.build();
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.build();
try {
client.newIndex("test1", settingsTest1, new HashMap<>())
.newIndex("test2", settingsTest2, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 1234; i++) {
client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
logger.info("refreshing");
client.refreshIndex("test1");
client.refreshIndex("test2");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setIndices("test1", "test2")
.setQuery(matchAllQuery());
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE)
.all();
IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal();
IndexingStats indexingStats = commonStats.getIndexing();
IndexingStats.Stats stats = indexingStats.getTotal();
logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
for (Map.Entry<Integer, IndexShardStats> me : indexStats.getIndexShards().entrySet()) {
IndexShardStats indexShardStats = me.getValue();
CommonStats commonShardStats = indexShardStats.getTotal();
logger.info("shard {} count = {}", me.getKey(),
commonShardStats.getIndexing().getTotal().getIndexCount());
}
}
try {
client.deleteIndex("test1")
.deleteIndex("test2");
} catch (Exception e) {
logger.error("delete index failed, ignored. Reason:", e);
}
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
@Test
public void testUpdateReplicaLevel() throws Exception {
long numberOfShards = 2;
int replicaLevel = 3;
// we need 3 nodes for replica level 3
startNode("2");
startNode("3");
Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", 0)
.build();
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.build();
try {
client.newIndex("replicatest", settings, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 12345; i++) {
client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS);
assertEquals(replicaLevel, client.getReplicaLevel("replicatest"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

33
elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java
Unescape Escape View File

@ -4,8 +4,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.http.ExtendedHttpClient;
@ -13,37 +14,39 @@ import org.xbib.elx.http.ExtendedHttpClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@Ignore
public class SmokeTest extends TestBase {
@Disabled
@ExtendWith(TestExtension.class)
class SmokeTest {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName());
private final TestExtension.Helper helper;
SmokeTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void smokeTest() throws Exception {
void smokeTest() throws Exception {
final ExtendedHttpClient client = ClientBuilder.builder()
.provider(ExtendedHttpClientProvider.class)
.put(helper.getHttpSettings())
.build();
try {
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
assertEquals(getClusterName(), client.getClusterName());
assertEquals(helper.getClusterName(), client.getClusterName());
client.checkMapping("test");
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.flush();
client.waitForRecovery("test", 10L, TimeUnit.SECONDS);
client.delete("test", "1");
client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel());
@ -51,10 +54,8 @@ public class SmokeTest extends TestBase {
client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
client.flush();
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());

198
elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java
Unescape Escape View File

@ -1,198 +0,0 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private AtomicInteger counter = new AtomicInteger();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
this.cluster = "test-helper-cluster-" + counter.incrementAndGet();
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("host", host)
.put("port", port)
.put("cluster.name", cluster)
.put("path.home", getHome())
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("discovery.zen.minimum_master_nodes", "1")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

236
elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java
Unescape Escape View File

@ -0,0 +1,236 @@
package org.xbib.elx.http.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getHttp().getAddress().publishAddress();
helper.httpHost = address.address().getHostName();
helper.httpPort = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
String httpHost;
int httpPort;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
Settings getHttpSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.put("host", httpHost)
.put("port", httpPort)
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

61
elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java
Unescape Escape View File

@ -1,10 +1,5 @@
package org.xbib.elx.node.test;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
@ -17,9 +12,10 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
@ -30,27 +26,28 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
public class ClientTest extends TestBase {
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 25000L;
private static final Long ACTIONS = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
@Before
public void startNodes() {
try {
super.startNodes();
startNode("2");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSingleDoc() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testSingleDoc() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
@ -73,8 +70,8 @@ public class ClientTest extends TestBase {
}
@Test
public void testNewIndex() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testNewIndex() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
@ -83,12 +80,12 @@ public class ClientTest extends TestBase {
}
@Test
public void testMapping() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testMapping() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.build();
XContentBuilder builder = jsonBuilder()
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
@ -108,9 +105,9 @@ public class ClientTest extends TestBase {
}
@Test
public void testRandomDocs() throws Exception {
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
@ -118,7 +115,8 @@ public class ClientTest extends TestBase {
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -140,12 +138,12 @@ public class ClientTest extends TestBase {
}
@Test
public void testThreadedRandomDocs() throws Exception {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
@ -164,7 +162,8 @@ public class ClientTest extends TestBase {
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});

30
elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java
Unescape Escape View File

@ -7,7 +7,8 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.node.ExtendedNodeClient;
@ -15,28 +16,37 @@ import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.*;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class DuplicateIDTest extends TestBase {
@ExtendWith(TestExtension.class)
class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 12345L;
private static final Long ACTIONS = 5L;
private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testDuplicateDocIDs() throws Exception {
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -47,7 +57,7 @@ public class DuplicateIDTest extends TestBase {
searchRequest.indices("test");
searchRequest.types("test");
searchRequest.source(builder);
long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);
} catch (NoNodeAvailableException e) {

25
elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java
Unescape Escape View File

@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
@ -19,17 +20,24 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class IndexPruneTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexPruneTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexPruneTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testPrune() throws IOException {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
@ -45,15 +53,12 @@ public class IndexPruneTest extends TestBase {
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();

28
elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java
Unescape Escape View File

@ -6,7 +6,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
@ -16,16 +17,23 @@ import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class IndexShiftTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexShiftTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexShiftTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testIndexShift() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void testIndexShift() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
@ -35,14 +43,13 @@ public class IndexShiftTest extends TestBase {
.build();
client.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
@ -63,7 +70,8 @@ public class IndexShiftTest extends TestBase {
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);

151
elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java
Unescape Escape View File

@ -1,151 +0,0 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.IndexingStats;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.node.ExtendedNodeClient;
import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@Ignore
public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName());
@Test
public void testReplicaLevel() throws Exception {
// we need nodes for replica levels
startNode("2");
startNode("3");
startNode("4");
Settings settingsTest1 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 3)
.build();
Settings settingsTest2 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 1)
.build();
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
client.newIndex("test1", settingsTest1, new HashMap<>())
.newIndex("test2", settingsTest2, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 1234; i++) {
client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
logger.info("refreshing");
client.refreshIndex("test1");
client.refreshIndex("test2");
SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
.setIndices("test1", "test2")
.setQuery(matchAllQuery());
long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE)
.all();
IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal();
IndexingStats indexingStats = commonStats.getIndexing();
IndexingStats.Stats stats = indexingStats.getTotal();
logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
for (Map.Entry<Integer, IndexShardStats> me : indexStats.getIndexShards().entrySet()) {
IndexShardStats indexShardStats = me.getValue();
CommonStats commonShardStats = indexShardStats.getTotal();
logger.info("shard {} count = {}", me.getKey(),
commonShardStats.getIndexing().getTotal().getIndexCount());
}
}
try {
client.deleteIndex("test1")
.deleteIndex("test2");
} catch (Exception e) {
logger.error("delete index failed, ignored. Reason:", e);
}
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
@Test
public void testUpdateReplicaLevel() throws Exception {
long numberOfShards = 2;
int replicaLevel = 3;
// we need 3 nodes for replica level 3
startNode("2");
startNode("3");
Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", 0)
.build();
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
client.newIndex("replicatest", settings, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 12345; i++) {
client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS);
assertEquals(replicaLevel, client.getReplicaLevel("replicatest"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

27
elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java
Unescape Escape View File

@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.node.ExtendedNodeClient;
@ -12,31 +13,35 @@ import org.xbib.elx.node.ExtendedNodeClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
public class SmokeTest extends TestBase {
@ExtendWith(TestExtension.class)
class SmokeTest {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
private final TestExtension.Helper helper;
SmokeTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void smokeTest() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(client("1"))
void smokeTest() throws Exception {
final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
.provider(ExtendedNodeClientProvider.class)
.build();
try {
assertEquals(getClusterName(), client.getClusterName());
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.update("test", "1", "{ \"name\" : \"Another name\"}");
client.delete("test", "1");
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test");
client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel());
@ -46,10 +51,8 @@ public class SmokeTest extends TestBase {
client.flush();
client.waitForResponses(30, TimeUnit.SECONDS);
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());

204
elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java
Unescape Escape View File

@ -1,204 +0,0 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
this.cluster = "test-cluster-" + System.getProperty("user.name");
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.all();
ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("host", host)
.put("port", port)
.put("cluster.name", cluster)
.put("path.home", getHome())
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("discovery.zen.minimum_master_nodes", "1")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

236
elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java
Unescape Escape View File

@ -0,0 +1,236 @@
package org.xbib.elx.node.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress();
helper.host = address.address().getHostName();
helper.port = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
String host;
int port;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
Settings getTransportSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("path.home", getHome())
.put("host", host)
.put("port", port)
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

59
elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java
Unescape Escape View File

@ -13,10 +13,11 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -27,36 +28,32 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class ClientTest extends TestBase {
@ExtendWith(TestExtension.class)
class ClientTest {
private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
private static final Long ACTIONS = 25000L;
private static final Long ACTIONS = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
@Before
public void startNodes() {
try {
super.startNodes();
startNode("2");
} catch (Throwable t) {
logger.error("startNodes failed", t);
}
private final TestExtension.Helper helper;
ClientTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testSingleDoc() throws Exception {
void testSingleDoc() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test");
@ -76,24 +73,24 @@ public class ClientTest extends TestBase {
}
@Test
public void testNewIndex() throws Exception {
void testNewIndex() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
client.newIndex("test");
client.close();
}
@Test
public void testMapping() throws Exception {
void testMapping() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
XContentBuilder builder = jsonBuilder()
XContentBuilder builder = JsonXContent.contentBuilder()
.startObject()
.startObject("doc")
.startObject("properties")
@ -113,18 +110,19 @@ public class ClientTest extends TestBase {
}
@Test
public void testRandomDocs() throws Exception {
void testRandomDocs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -152,7 +150,7 @@ public class ClientTest extends TestBase {
}
@Test
public void testThreadedRandomDocs() throws Exception {
void testThreadedRandomDocs() throws Exception {
int maxthreads = Runtime.getRuntime().availableProcessors();
Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
final Long actions = ACTIONS;
@ -162,7 +160,7 @@ public class ClientTest extends TestBase {
.put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
.put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
@ -177,7 +175,8 @@ public class ClientTest extends TestBase {
for (int i = 0; i < maxthreads; i++) {
executorService.execute(() -> {
for (int i1 = 0; i1 < actions; i1++) {
client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", null, false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
latch.countDown();
});

31
elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java
Unescape Escape View File

@ -8,7 +8,8 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.common.Parameters;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -16,30 +17,38 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class DuplicateIDTest extends TestBase {
@ExtendWith(TestExtension.class)
class DuplicateIDTest {
private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
private static final Long ACTIONS = 12345L;
private static final Long ACTIONS = 5L;
private final TestExtension.Helper helper;
DuplicateIDTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testDuplicateDocIDs() throws Exception {
void testDuplicateDocIDs() throws Exception {
long numactions = ACTIONS;
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
client.newIndex("test");
for (int i = 0; i < ACTIONS; i++) {
client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
@ -52,7 +61,7 @@ public class DuplicateIDTest extends TestBase {
searchRequest.types("test");
searchRequest.source(builder);
SearchResponse searchResponse =
client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("hits = {}", hits);
assertTrue(hits < ACTIONS);

25
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java
Unescape Escape View File

@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexPruneResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -19,19 +20,26 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class IndexPruneTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexPruneTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexPruneTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testPrune() throws IOException {
void testPrune() throws IOException {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
@ -46,15 +54,12 @@ public class IndexPruneTest extends TestBase {
client.shiftIndex("test", "test3", Collections.emptyList());
client.newIndex("test4", settings);
client.shiftIndex("test", "test4", Collections.emptyList());
IndexPruneResult indexPruneResult =
client.pruneIndex("test", "test4", 2, 2, true);
assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
List<Boolean> list = new ArrayList<>();
for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();

40
elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java
Unescape Escape View File

@ -6,31 +6,36 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexShiftResult;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@Ignore
public class IndexShiftTest extends TestBase {
@ExtendWith(TestExtension.class)
class IndexShiftTest {
private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName());
private final TestExtension.Helper helper;
IndexShiftTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
public void testIndexShift() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder(client("1"))
void testIndexShift() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
Settings settings = Settings.builder()
@ -39,39 +44,35 @@ public class IndexShiftTest extends TestBase {
.build();
client.newIndex("test1234", settings);
for (int i = 0; i < 1; i++) {
client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test1234", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
IndexShiftResult indexShiftResult =
client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c"));
assertTrue(indexShiftResult.getNewAliases().contains("a"));
assertTrue(indexShiftResult.getNewAliases().contains("b"));
assertTrue(indexShiftResult.getNewAliases().contains("c"));
assertTrue(indexShiftResult.getMovedAliases().isEmpty());
Map<String, String> aliases = client.getAliases("test1234");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
String resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
assertTrue(aliases.containsKey("c"));
assertTrue(aliases.containsKey("test"));
client.newIndex("test5678", settings);
for (int i = 0; i < 1; i++) {
client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}");
client.index("test5678", helper.randomString(1), false,
"{ \"name\" : \"" + helper.randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
(request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
.index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
@ -82,7 +83,6 @@ public class IndexShiftTest extends TestBase {
assertTrue(indexShiftResult.getMovedAliases().contains("a"));
assertTrue(indexShiftResult.getMovedAliases().contains("b"));
assertTrue(indexShiftResult.getMovedAliases().contains("c"));
aliases = client.getAliases("test5678");
assertTrue(aliases.containsKey("a"));
assertTrue(aliases.containsKey("b"));
@ -90,7 +90,6 @@ public class IndexShiftTest extends TestBase {
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
resolved = client.resolveAlias("test");
aliases = client.getAliases(resolved);
assertTrue(aliases.containsKey("a"));
@ -99,7 +98,6 @@ public class IndexShiftTest extends TestBase {
assertTrue(aliases.containsKey("d"));
assertTrue(aliases.containsKey("e"));
assertTrue(aliases.containsKey("f"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {

161
elx-transport/src/test/java/org/xbib/elx/transport/test/ReplicaTest.java
Unescape Escape View File

@ -1,161 +0,0 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexShardStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.shard.IndexingStats;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Ignore;
import org.junit.Test;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@Ignore
public class ReplicaTest extends TestBase {
private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName());
@Test
public void testReplicaLevel() throws Exception {
// we need nodes for replica levels
startNode("2");
startNode("3");
startNode("4");
Settings settingsTest1 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 3)
.build();
Settings settingsTest2 = Settings.builder()
.put("index.number_of_shards", 2)
.put("index.number_of_replicas", 1)
.build();
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.build();
try {
client.newIndex("test1", settingsTest1, new HashMap<>())
.newIndex("test2", settingsTest2, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 1234; i++) {
client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
for (int i = 0; i < 1234; i++) {
client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
logger.info("refreshing");
client.refreshIndex("test1");
client.refreshIndex("test2");
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(0);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices("test1", "test2");
searchRequest.source(builder);
SearchResponse searchResponse =
client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
long hits = searchResponse.getHits().getTotalHits();
logger.info("query total hits={}", hits);
assertEquals(2468, hits);
IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
indicesStatsRequest.all();
IndicesStatsResponse response =
client.getClient().execute(IndicesStatsAction.INSTANCE, indicesStatsRequest).actionGet();
for (Map.Entry<String, IndexStats> m : response.getIndices().entrySet()) {
IndexStats indexStats = m.getValue();
CommonStats commonStats = indexStats.getTotal();
IndexingStats indexingStats = commonStats.getIndexing();
IndexingStats.Stats stats = indexingStats.getTotal();
logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount());
for (Map.Entry<Integer, IndexShardStats> me : indexStats.getIndexShards().entrySet()) {
IndexShardStats indexShardStats = me.getValue();
CommonStats commonShardStats = indexShardStats.getTotal();
logger.info("shard {} count = {}", me.getKey(),
commonShardStats.getIndexing().getTotal().getIndexCount());
}
}
try {
client.deleteIndex("test1")
.deleteIndex("test2");
} catch (Exception e) {
logger.error("delete index failed, ignored. Reason:", e);
}
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
@Test
public void testUpdateReplicaLevel() throws Exception {
long numberOfShards = 2;
int replicaLevel = 3;
// we need 3 nodes for replica level 3
startNode("2");
startNode("3");
Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", 0)
.build();
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.build();
try {
client.newIndex("replicatest", settings, new HashMap<>());
client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
for (int i = 0; i < 12345; i++) {
client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}");
}
client.flush();
client.waitForResponses(30L, TimeUnit.SECONDS);
client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS);
assertEquals(replicaLevel, client.getReplicaLevel("replicatest"));
} catch (NoNodeAvailableException e) {
logger.warn("skipping, no node available");
} finally {
client.close();
if (client.getBulkController().getLastBulkError() != null) {
logger.error("error", client.getBulkController().getLastBulkError());
}
assertNull(client.getBulkController().getLastBulkError());
}
}
}

25
elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java
Unescape Escape View File

@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.client.transport.NoNodeAvailableException;
import org.elasticsearch.common.settings.Settings;
import org.junit.Test;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.xbib.elx.api.IndexDefinition;
import org.xbib.elx.common.ClientBuilder;
import org.xbib.elx.transport.ExtendedTransportClient;
@ -12,22 +13,28 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
public class SmokeTest extends TestBase {
@ExtendWith(TestExtension.class)
class SmokeTest {
private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
@Test
public void smokeTest() throws Exception {
private final TestExtension.Helper helper;
SmokeTest(TestExtension.Helper helper) {
this.helper = helper;
}
@Test
void smokeTest() throws Exception {
final ExtendedTransportClient client = ClientBuilder.builder()
.provider(ExtendedTransportClientProvider.class)
.put(getTransportSettings())
.put(helper.getTransportSettings())
.build();
try {
assertEquals(getClusterName(), client.getClusterName());
assertEquals(helper.getClusterName(), client.getClusterName());
client.newIndex("test");
client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
client.update("test", "1", "{ \"name\" : \"Another name\"}");
@ -36,7 +43,6 @@ public class SmokeTest extends TestBase {
client.waitForResponses(30, TimeUnit.SECONDS);
client.checkMapping("test");
client.deleteIndex("test");
IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder()
.build());
assertEquals(0, indexDefinition.getReplicaLevel());
@ -47,7 +53,6 @@ public class SmokeTest extends TestBase {
client.updateReplicaLevel(indexDefinition, 2);
int replica = client.getReplicaLevel(indexDefinition);
assertEquals(2, replica);
client.deleteIndex(indexDefinition);
assertEquals(0, client.getBulkMetric().getFailed().getCount());
assertEquals(4, client.getBulkMetric().getSucceeded().getCount());

204
elx-transport/src/test/java/org/xbib/elx/transport/test/TestBase.java
Unescape Escape View File

@ -1,204 +0,0 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
public class TestBase {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private Map<String, Node> nodes = new HashMap<>();
private Map<String, AbstractClient> clients = new HashMap<>();
private String cluster;
private String host;
private int port;
@Before
public void startNodes() {
try {
logger.info("starting");
this.cluster = "test-cluster-" + System.getProperty("user.name");
startNode("1");
findNodeAddress();
try {
ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
clusterStateRequest.all();
ClusterStateResponse clusterStateResponse =
client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
logger.info("host = {} port = {}", host, port);
} catch (Throwable t) {
logger.error(t.getMessage(), t);
}
}
@After
public void stopNodes() {
try {
closeNodes();
} catch (Exception e) {
logger.error("can not close nodes", e);
} finally {
try {
deleteFiles();
logger.info("data files wiped");
Thread.sleep(2000L); // let OS commit changes
} catch (IOException e) {
logger.error(e.getMessage(), e);
} catch (InterruptedException e) {
// ignore
}
}
}
protected Settings getTransportSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("path.home", getHome())
.put("host", host)
.put("port", port)
.build();
}
protected Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("discovery.zen.minimum_master_nodes", "1")
.put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME)
.put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here
.put("path.home", getHome())
.build();
}
protected static String getHome() {
return System.getProperty("path.home", System.getProperty("user.dir"));
}
protected void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
protected AbstractClient client(String id) {
return clients.get(id);
}
protected String getClusterName() {
return cluster;
}
protected String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private void closeNodes() throws IOException {
logger.info("closing all clients");
for (AbstractClient client : clients.values()) {
client.close();
}
clients.clear();
logger.info("closing all nodes");
for (Node node : nodes.values()) {
if (node != null) {
node.close();
}
}
nodes.clear();
logger.info("all nodes closed");
}
private void findNodeAddress() {
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress()
.publishAddress();
host = address.address().getHostName();
port = address.address().getPort();
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
private static void deleteFiles() throws IOException {
Path directory = Paths.get(getHome() + "/data");
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}

236
elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java
Unescape Escape View File

@ -0,0 +1,236 @@
package org.xbib.elx.transport.test;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest;
import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
import org.elasticsearch.analysis.common.CommonAnalysisPlugin;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.client.support.AbstractClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.NodeValidationException;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.transport.netty4.Netty4Plugin;
import org.junit.jupiter.api.extension.AfterEachCallback;
import org.junit.jupiter.api.extension.BeforeEachCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ParameterContext;
import org.junit.jupiter.api.extension.ParameterResolutionException;
import org.junit.jupiter.api.extension.ParameterResolver;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Junit 5 extension for testing Elasticsearch.
* The extension will be instantiated as a singleton.
* For parallel test method executions, for example in gradle, it requires a helper class
* to ensure different ES homes/clusters for each run.
*/
public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback {
private static final Logger logger = LogManager.getLogger("test");
private static final Random random = new Random();
private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray();
private static final String key = "es-instance-";
private static final AtomicInteger count = new AtomicInteger(0);
private static final ExtensionContext.Namespace ns =
ExtensionContext.Namespace.create(TestExtension.class);
@Override
public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
return parameterContext.getParameter().getType().equals(Helper.class);
}
@Override
public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext)
throws ParameterResolutionException {
// initialize new helper here, increase counter
return extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class);
}
@Override
public void beforeEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
logger.info("starting cluster with helper " + helper + " at " + helper.getHome());
helper.startNode("1");
NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true);
NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet();
TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress();
helper.host = address.address().getHostName();
helper.port = address.address().getPort();
try {
ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE,
new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN)
.timeout(TimeValue.timeValueSeconds(30))).actionGet();
if (healthResponse != null && healthResponse.isTimedOut()) {
throw new IOException("cluster state is " + healthResponse.getStatus().name()
+ ", from here on, everything will fail!");
}
} catch (ElasticsearchTimeoutException e) {
throw new IOException("cluster does not respond to health request, cowardly refusing to continue");
}
ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all();
ClusterStateResponse clusterStateResponse =
helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
logger.info("cluster name = {}", clusterStateResponse.getClusterName().value());
}
@Override
public void afterEach(ExtensionContext extensionContext) throws Exception {
Helper helper = extensionContext.getParent().get().getStore(ns)
.getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class);
closeNodes(helper);
deleteFiles(Paths.get(helper.getHome()));
logger.info("data files wiped: " + helper.getHome());
Thread.sleep(2000L); // let OS commit changes
}
private void closeNodes(Helper helper) throws IOException {
logger.info("closing all clients");
for (AbstractClient client : helper.clients.values()) {
client.close();
}
logger.info("closing all nodes");
for (Node node : helper.nodes.values()) {
if (node != null) {
node.close();
}
}
logger.info("all nodes closed");
}
private static void deleteFiles(Path directory) throws IOException {
if (Files.exists(directory)) {
Files.walkFileTree(directory, new SimpleFileVisitor<>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
Files.delete(dir);
return FileVisitResult.CONTINUE;
}
});
}
}
private Helper create() {
Helper helper = new Helper();
helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8));
helper.setClusterName("test-cluster-" + helper.randomString(8));
logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome());
return helper;
}
class Helper {
String home;
String cluster;
String host;
int port;
Map<String, Node> nodes = new HashMap<>();
Map<String, AbstractClient> clients = new HashMap<>();
void setHome(String home) {
this.home = home;
}
String getHome() {
return home;
}
void setClusterName(String cluster) {
this.cluster = cluster;
}
String getClusterName() {
return cluster;
}
Settings getNodeSettings() {
return Settings.builder()
.put("cluster.name", getClusterName())
.put("path.home", getHome())
.build();
}
Settings getTransportSettings() {
return Settings.builder()
.put("cluster.name", cluster)
.put("path.home", getHome())
.put("host", host)
.put("port", port)
.build();
}
void startNode(String id) throws NodeValidationException {
buildNode(id).start();
}
ElasticsearchClient client(String id) {
return clients.get(id);
}
String randomString(int len) {
final char[] buf = new char[len];
final int n = numbersAndLetters.length - 1;
for (int i = 0; i < buf.length; i++) {
buf[i] = numbersAndLetters[random.nextInt(n)];
}
return new String(buf);
}
private Node buildNode(String id) {
Settings nodeSettings = Settings.builder()
.put(getNodeSettings())
.put("node.name", id)
.build();
List<Class<? extends Plugin>> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class);
Node node = new MockNode(nodeSettings, plugins);
AbstractClient client = (AbstractClient) node.client();
nodes.put(id, node);
clients.put(id, client);
return node;
}
}
}

10
gradle.properties
Unescape Escape View File

@ -1,17 +1,17 @@
group = org.xbib
name = elx
version = 6.3.2.1
version = 6.3.2.2
profile = default
release = 0
elasticsearch-server.version = 6.3.2.2
elasticsearch-server.version = 6.3.2.3
log4j.version = 2.11.1
xbib-metrics.version = 1.2.0
xbib-netty-http.version = 4.1.33.0
xbib-netty-http.version = 4.1.35.0
# test
junit.version = 4.12
wagon.version = 3.0.0
junit.version = 5.4.2
wagon.version = 3.3.2
asciidoclet.version = 1.6.0.0
org.gradle.warning.mode = all

BIN
gradle/wrapper/gradle-wrapper.jar vendored
View File

Binary file not shown.

3
gradle/wrapper/gradle-wrapper.properties vendored
Unescape Escape View File

@ -1,6 +1,5 @@
#Fri Feb 15 10:39:20 CET 2019
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip

18
gradlew vendored
Unescape Escape View File

@ -1,5 +1,21 @@
#!/usr/bin/env sh
#
# Copyright 2015 the original author or authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##############################################################################
##
## Gradle start up script for UN*X
@ -28,7 +44,7 @@ APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m"'
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"

18
gradlew.bat vendored
Unescape Escape View File

@ -1,3 +1,19 @@
@rem
@rem Copyright 2015 the original author or authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS="-Xmx64m"
set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome

Write Preview
Loading…
Cancel
Save