diff --git a/build.gradle b/build.gradle index 3e661f3..b12951a 100644 --- a/build.gradle +++ b/build.gradle @@ -16,12 +16,12 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s System.getProperty("java.vm.version"), System.getProperty("java.vm.vendor"), System.getProperty("java.vm.name"), - gradle.gradleVersion, - GroovySystem.getVersion(), + gradle.gradleVersion, + GroovySystem.getVersion(), JavaVersion.current() if (JavaVersion.current() < JavaVersion.VERSION_11) { - throw new GradleException("This build must be run with java 11 or higher") + throw new GradleException("This build must be run with java 11") } subprojects { @@ -39,9 +39,11 @@ subprojects { } dependencies { - testCompile "junit:junit:${project.property('junit.version')}" - testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" - testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" + testImplementation "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" + testImplementation "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}" + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" } @@ -63,21 +65,35 @@ subprojects { } test { - jvmArgs =[ - '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', - '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED' - ] + enabled = true + useJUnitPlatform() + // we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery + doFirst { + + jvmArgs = [ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] + } + systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' + systemProperty 'path.home', "${project.buildDir}" systemProperty 'jna.debug_load', 'true' testLogging { - showStandardStreams = true - exceptionFormat = 'full' + events 'PASSED', 'FAILED', 'SKIPPED' + } + afterSuite { desc, result -> + if (!desc.parent) { + println "\nTest result: ${result.resultType}" + println "Test summary: ${result.testCount} tests, " + + "${result.successfulTestCount} succeeded, " + + "${result.failedTestCount} failed, " + + "${result.skippedTestCount} skipped" + } } } clean { - delete "data" - delete "logs" delete "out" } @@ -94,18 +110,18 @@ subprojects { }*/ task javadocJar(type: Jar, dependsOn: javadoc) { - classifier 'javadoc' + archiveClassifier.set('javadoc') } task sourcesJar(type: Jar, dependsOn: classes) { from sourceSets.main.allSource - classifier 'sources' + archiveClassifier.set('sources') } artifacts { archives javadocJar, sourcesJar } - + if (project.hasProperty('signing.keyId')) { signing { sign configurations.archives @@ -164,35 +180,3 @@ subprojects { } } } - -/*asciidoctor { - attributes toc: 'left', - doctype: 'book', - icons: 'font', - encoding: 'utf-8', - sectlink: true, - sectanchors: true, - linkattrs: true, - imagesdir: 'img', - 'source-highlighter': 'coderay' -}*/ - -/* -task aggregatedJavadoc(type: Javadoc) { - group = 'aggregation' - description = 'Generates aggregated Javadoc API documentation.' - title = "$description $version API" - destinationDir = file("$buildDir/docs/javadoc") - def sourceProjects = subprojects.findAll { - it.plugins.hasPlugin('java') || it.plugins.hasPlugin('groovy') - } - source sourceProjects.collect { - it.sourceSets.main.allJava - } - classpath = files(sourceProjects.collect { - it.sourceSets.main.runtimeClasspath - }) - //options.overview = 'gradle/api/overview.html' - options.showFromProtected() -} -*/ diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index e08f90a..f3fda84 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -398,9 +398,9 @@ public interface ExtendedClient extends Flushable, Closeable { String resolveMostRecentIndex(String alias); /** - * Get all index filters. + * Get all index aliases. * @param index the index - * @return map of index filters + * @return map of index aliases */ Map getAliases(String index); diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java new file mode 100644 index 0000000..2ecb857 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java @@ -0,0 +1,25 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; + +public interface ReadClient { + + ActionFuture get(GetRequest getRequest); + + void get(GetRequest request, ActionListener listener); + + ActionFuture multiGet(MultiGetRequest request); + + void multiGet(MultiGetRequest request, ActionListener listener); + + ActionFuture search(SearchRequest request); + + void search(SearchRequest request, ActionListener listener); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java new file mode 100644 index 0000000..b815af7 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java @@ -0,0 +1,6 @@ +package org.xbib.elx.api; + +public interface ReadClientProvider { + + C getReadClient(); +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 92d2bb6..646e4db 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -454,7 +454,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() .indices(index) - .waitForActiveShards(shards).timeout(timeout); + .waitForActiveShards(shards) + .timeout(timeout); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index 30d5b52..873145b 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -83,12 +82,12 @@ public class DefaultBulkController implements BulkController { maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); } this.bulkListener = new BulkListener(); - DefaultBulkProcessor.Builder builder = DefaultBulkProcessor.builder((Client) client.getClient(), bulkListener) + this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) .setConcurrentRequests(maxConcurrentRequests) .setFlushInterval(flushIngestInterval) - .setBulkSize(maxVolumePerRequest); - this.bulkProcessor = builder.build(); + .setBulkSize(maxVolumePerRequest) + .build(); this.active.set(true); } @@ -115,9 +114,7 @@ public class DefaultBulkController implements BulkController { @Override public void index(IndexRequest indexRequest) { - if (!active.get()) { - throw new IllegalStateException("inactive"); - } + ensureActiveAndBulk(); try { if (bulkMetric != null) { bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); @@ -226,6 +223,18 @@ public class DefaultBulkController implements BulkController { } } + private void ensureActiveAndBulk() { + if (!active.get()) { + throw new IllegalStateException("inactive"); + } + if (bulkProcessor == null) { + throw new UnsupportedOperationException("bulk processor not present"); + } + if (bulkListener == null) { + throw new UnsupportedOperationException("bulk listener not present"); + } + } + private class BulkListener implements DefaultBulkProcessor.Listener { private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 99a72ec..0abebc6 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -8,7 +8,8 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -16,6 +17,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.xbib.elx.api.BulkProcessor; +import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -47,7 +49,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private volatile boolean closed; - private DefaultBulkProcessor(Client client, Listener listener, String name, int concurrentRequests, + private DefaultBulkProcessor(ElasticsearchClient client, Listener listener, String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) { this.executionIdGen = new AtomicLong(); this.closed = false; @@ -59,7 +61,7 @@ public class DefaultBulkProcessor implements BulkProcessor { new AsyncBulkRequestHandler(client, listener, concurrentRequests); if (flushInterval != null) { this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, - EsExecutors.daemonThreadFactory(client.settings(), + EsExecutors.daemonThreadFactory(Settings.EMPTY, name != null ? "[" + name + "]" : "" + "bulk_processor")); this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); @@ -71,10 +73,9 @@ public class DefaultBulkProcessor implements BulkProcessor { } } - public static Builder builder(Client client, Listener listener) { - if (client == null) { - throw new NullPointerException("The client you specified while building a BulkProcessor is null"); - } + public static Builder builder(ElasticsearchClient client, Listener listener) { + Objects.requireNonNull(client, "The client you specified while building a BulkProcessor is null"); + Objects.requireNonNull(listener, "A listener for the BulkProcessor is required but null"); return new Builder(client, listener); } @@ -87,6 +88,7 @@ public class DefaultBulkProcessor implements BulkProcessor { */ @Override public synchronized boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException { + Objects.requireNonNull(unit, "A time unit is required for awaitFlush() but null"); if (closed) { return true; } @@ -95,7 +97,7 @@ public class DefaultBulkProcessor implements BulkProcessor { execute(); } // wait for all bulk responses - return this.bulkRequestHandler.close(timeout, unit); + return bulkRequestHandler.close(timeout, unit); } /** @@ -115,18 +117,19 @@ public class DefaultBulkProcessor implements BulkProcessor { */ @Override public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + Objects.requireNonNull(unit, "A time unit is required for awaitCLose() but null"); if (closed) { return true; } closed = true; - if (this.scheduledFuture != null) { - FutureUtils.cancel(this.scheduledFuture); + if (scheduledFuture != null) { + FutureUtils.cancel(scheduledFuture); this.scheduler.shutdown(); } if (bulkRequest.numberOfActions() > 0) { execute(); } - return this.bulkRequestHandler.close(timeout, unit); + return bulkRequestHandler.close(timeout, unit); } /** @@ -215,8 +218,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private boolean isOverTheLimit() { return bulkActions != -1 && bulkRequest.numberOfActions() >= bulkActions || - bulkSize != -1 && - bulkRequest.estimatedSizeInBytes() >= bulkSize; + bulkSize != -1 && bulkRequest.estimatedSizeInBytes() >= bulkSize; } /** @@ -224,7 +226,7 @@ public class DefaultBulkProcessor implements BulkProcessor { */ public static class Builder { - private final Client client; + private final ElasticsearchClient client; private final Listener listener; @@ -245,7 +247,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @param client the client * @param listener the listener */ - Builder(Client client, Listener listener) { + Builder(ElasticsearchClient client, Listener listener) { this.client = client; this.listener = listener; } @@ -339,11 +341,12 @@ public class DefaultBulkProcessor implements BulkProcessor { private static class SyncBulkRequestHandler implements BulkRequestHandler { - private final Client client; + private final ElasticsearchClient client; private final DefaultBulkProcessor.Listener listener; - SyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener) { + SyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener) { + Objects.requireNonNull(listener, "A listener is required for SyncBulkRequestHandler but null"); this.client = client; this.listener = listener; } @@ -371,7 +374,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private static class AsyncBulkRequestHandler implements BulkRequestHandler { - private final Client client; + private final ElasticsearchClient client; private final DefaultBulkProcessor.Listener listener; @@ -379,7 +382,8 @@ public class DefaultBulkProcessor implements BulkProcessor { private final int concurrentRequests; - private AsyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { + private AsyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { + Objects.requireNonNull(listener, "A listener is required for AsyncBulkRequestHandler but null"); this.client = client; this.listener = listener; this.concurrentRequests = concurrentRequests; diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java index c20ee45..63491bf 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java @@ -1,19 +1,19 @@ package org.xbib.elx.common.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.Strings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.util.Collections; import java.util.Iterator; @@ -22,44 +22,54 @@ import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; -public class AliasTest extends TestBase { +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class AliasTest { private static final Logger logger = LogManager.getLogger(AliasTest.class.getName()); + private final TestExtension.Helper helper; + + AliasTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testAlias() { - Client client = client("1"); + void testAlias() { + ElasticsearchClient client = helper.client("1"); CreateIndexRequest indexRequest = new CreateIndexRequest("test"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); // put alias IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); - String[] indices = new String[]{"test"}; - String[] aliases = new String[]{"test_alias"}; + String[] indices = new String[] { "test" }; + String[] aliases = new String[] { "test_alias" }; IndicesAliasesRequest.AliasActions aliasAction = new IndicesAliasesRequest.AliasActions(IndicesAliasesRequest.AliasActions.Type.ADD) .indices(indices) .aliases(aliases); indicesAliasesRequest.addAliasAction(aliasAction); - client.admin().indices().aliases(indicesAliasesRequest).actionGet(); + client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); // get alias GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY); long t0 = System.nanoTime(); - GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(getAliasesRequest).actionGet(); + GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); long t1 = (System.nanoTime() - t0) / 1000000; logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1); assertTrue(t1 >= 0); } @Test - public void testMostRecentIndex() { - Client client = client("1"); + void testMostRecentIndex() { + ElasticsearchClient client = helper.client("1"); String alias = "test"; CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); indexRequest = new CreateIndexRequest("test20160102"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); indexRequest = new CreateIndexRequest("test20160103"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); String[] indices = new String[]{"test20160101", "test20160102", "test20160103"}; String[] aliases = new String[]{alias}; @@ -68,7 +78,7 @@ public class AliasTest extends TestBase { .indices(indices) .aliases(aliases); indicesAliasesRequest.addAliasAction(aliasAction); - client.admin().indices().aliases(indicesAliasesRequest).actionGet(); + client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); getAliasesRequest.aliases(alias); @@ -90,5 +100,4 @@ public class AliasTest extends TestBase { assertEquals("test20160101", it.next()); logger.info("success: result={}", result); } - } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java deleted file mode 100644 index 2999175..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.xbib.elx.common.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -@Ignore -public class ClusterBlockTest extends TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - @Before - public void startNodes() { - try { - setClusterName("test-cluster-" + System.getProperty("user.name")); - startNode("1"); - // do not wait for green health state - logger.info("ready"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } - } - - @Override - protected Settings getNodeSettings() { - return Settings.builder() - .put(super.getNodeSettings()) - .put("discovery.zen.minimum_master_nodes", 2) // block until we have two nodes - .build(); - } - - @Test(expected = ClusterBlockException.class) - public void testClusterBlock() throws Exception { - Client client = client("1"); - XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("field1", "value1").endObject(); - IndexRequestBuilder irb = client.prepareIndex("test", "test", "1").setSource(builder); - BulkRequestBuilder brb = client.prepareBulk(); - brb.add(irb); - brb.execute().actionGet(); - } -} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java index cbe7972..e21817c 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java @@ -1,18 +1,18 @@ package org.xbib.elx.common.test; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.MockExtendedClient; import org.xbib.elx.common.MockExtendedClientProvider; import java.io.IOException; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; -public class MockExtendedClientProviderTest { +class MockExtendedClientProviderTest { @Test - public void testMockExtendedProvider() throws IOException { + void testMockExtendedProvider() throws IOException { MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); assertNotNull(client); } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java index b83aa8a..48df8a0 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java @@ -7,9 +7,9 @@ import org.elasticsearch.plugins.Plugin; import java.util.List; -public class MockNode extends Node { +class MockNode extends Node { - public MockNode(Settings settings, List> classpathPlugins) { + MockNode(Settings settings, List> classpathPlugins) { super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java index 7933343..ef44d74 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java @@ -2,20 +2,22 @@ package org.xbib.elx.common.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import java.net.InetAddress; import java.net.NetworkInterface; import java.util.Collections; import java.util.Enumeration; -public class NetworkTest { +// walk over all found interfaces (this is slow - multicast/pings are performed) +@Disabled +class NetworkTest { private static final Logger logger = LogManager.getLogger(NetworkTest.class); @Test - public void testNetwork() throws Exception { - // walk over all found interfaces (this is slow - multicast/pings are performed) + void testNetwork() throws Exception { Enumeration nets = NetworkInterface.getNetworkInterfaces(); for (NetworkInterface netint : Collections.list(nets)) { System.out.println("checking network interface = " + netint.getName()); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index 6e23f0b..d1a4412 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -1,25 +1,36 @@ package org.xbib.elx.common.test; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class SearchTest extends TestBase { +@ExtendWith(TestExtension.class) +class SearchTest { + + private final TestExtension.Helper helper; + + SearchTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testSearch() throws Exception { - Client client = client("1"); + void testSearch() throws Exception { + ElasticsearchClient client = helper.client("1"); BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); for (int i = 0; i < 1000; i++) { IndexRequest indexRequest = new IndexRequest("pages", "row") @@ -39,18 +50,20 @@ public class SearchTest extends TestBase { .endObject()); builder.add(indexRequest); } - client.bulk(builder.request()).actionGet(); - client.admin().indices().refresh(new RefreshRequest()).actionGet(); - - for (int i = 0; i < 100; i++) { + client.execute(BulkAction.INSTANCE, builder.request()).actionGet(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); + for (int i = 0; i < 1; i++) { QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234); - SearchRequestBuilder requestBuilder = client.prepareSearch() - .setIndices("pages") - .setTypes("row") - .setQuery(queryStringBuilder) - .addSort("rowcount", SortOrder.DESC) - .setFrom(i * 10).setSize(10); - SearchResponse searchResponse = requestBuilder.execute().actionGet(); + SearchSourceBuilder searchSource = new SearchSourceBuilder(); + searchSource.query(queryStringBuilder); + searchSource.sort("rowcount", SortOrder.DESC); + searchSource.from(i * 10); + searchSource.size(10); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("pages"); + searchRequest.types("row"); + searchRequest.source(searchSource); + SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); assertTrue(searchResponse.getHits().getTotalHits() > 0); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java index 5d5bc4c..4c2e7ba 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java @@ -1,7 +1,5 @@ package org.xbib.elx.common.test; -import static org.junit.Assert.assertEquals; - import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; @@ -10,7 +8,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.settings.Settings; @@ -18,16 +15,26 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +@ExtendWith(TestExtension.class) +class SimpleTest { -public class SimpleTest extends TestBase { + private final TestExtension.Helper helper; + + SimpleTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void test() throws Exception { + void testSimple() throws Exception { try { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices("test"); - client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); + helper.client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); } catch (IndexNotFoundException e) { // ignore if index not found } @@ -38,24 +45,23 @@ public class SimpleTest extends TestBase { .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); createIndexRequest.index("test").settings(indexSettings); - client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + helper.client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); IndexRequest indexRequest = new IndexRequest(); indexRequest.index("test").type("test").id("1") .source(XContentFactory.jsonBuilder().startObject().field("field", "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()); - client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); + helper.client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); RefreshRequest refreshRequest = new RefreshRequest(); refreshRequest.indices("test"); - client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); + helper.client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchQuery("field", "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8")); SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("test").types("test"); searchRequest.source(builder); - String doc = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet() + String doc = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet() .getHits().getAt(0).getSourceAsString(); - assertEquals(doc, "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}"); } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java deleted file mode 100644 index 681e4b8..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java +++ /dev/null @@ -1,207 +0,0 @@ -package org.xbib.elx.common.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; -import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeValidationException; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -public class TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - private static final Random random = new Random(); - - private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String cluster; - - private String host; - - private int port; - - @Before - public void startNodes() { - try { - logger.info("starting"); - setClusterName("test-cluster-" + System.getProperty("user.name")); - startNode("1"); - findNodeAddress(); - try { - ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) - .timeout(TimeValue.timeValueSeconds(30))).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); - } - ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.all(); - ClusterStateResponse clusterStateResponse = - client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); - logger.info("host = {} port = {}", host, port); - } catch (Throwable t) { - logger.error(t.getMessage(), t); - } - } - - @After - public void stopNodes() { - try { - closeNodes(); - } catch (Exception e) { - logger.error("can not close nodes", e); - } finally { - try { - deleteFiles(); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected Settings getTransportSettings() { - return Settings.builder() - .put("host", host) - .put("port", port) - .put("cluster.name", cluster) - .put("path.home", getHome()) - .build(); - } - - protected Settings getNodeSettings() { - return Settings.builder() - .put("cluster.name", cluster) - .put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("path.home", getHome()) - .build(); - } - - protected static String getHome() { - return System.getProperty("path.home", System.getProperty("user.dir")); - } - - protected void startNode(String id) throws NodeValidationException { - buildNode(id).start(); - } - - protected AbstractClient client(String id) { - return clients.get(id); - } - - protected void setClusterName(String cluster) { - this.cluster = cluster; - } - - protected String getClusterName() { - return cluster; - } - - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - private void closeNodes() throws IOException { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - - private void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress() - .publishAddress(); - host = address.address().getHostName(); - port = address.address().getPort(); - } - - private Node buildNode(String id) { - Settings nodeSettings = Settings.builder() - .put(getNodeSettings()) - .put("node.name", id) - .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); - Node node = new MockNode(nodeSettings, plugins); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - logger.info("clients={}", clients); - return node; - } - - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } -} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java new file mode 100644 index 0000000..5439034 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java @@ -0,0 +1,224 @@ +package org.xbib.elx.common.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.analysis.common.CommonAnalysisPlugin; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Junit 5 extension for testing Elasticsearch. + * The extension will be instantiated as a singleton. + * For parallel test method executions, for example in gradle, it requires a helper class + * to ensure different ES homes/clusters for each run. + */ +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private static final String key = "es-instance-"; + + private static final AtomicInteger count = new AtomicInteger(0); + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress(); + String host = address.address().getHostName(); + int port = address.address().getPort(); + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome())); + logger.info("data files wiped: " + helper.getHome()); + Thread.sleep(2000L); // let OS commit changes + } + + private void closeNodes(Helper helper) throws IOException { + logger.info("closing all clients"); + for (AbstractClient client : helper.clients.values()) { + client.close(); + } + logger.info("closing all nodes"); + for (Node node : helper.nodes.values()) { + if (node != null) { + node.close(); + } + } + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; + } + + class Helper { + + String home; + + String cluster; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } + + Settings getNodeSettings() { + return Settings.builder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } + + void startNode(String id) throws NodeValidationException { + buildNode(id).start(); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + private Node buildNode(String id) { + Settings nodeSettings = Settings.builder() + .put(getNodeSettings()) + .put("node.name", id) + .build(); + List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + Node node = new MockNode(nodeSettings, plugins); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + return node; + } + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java index be7ef4d..0bc9b0f 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java @@ -6,43 +6,52 @@ import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.io.IOException; -public class WildcardTest extends TestBase { +@ExtendWith(TestExtension.class) +class WildcardTest { + + private final TestExtension.Helper helper; + + WildcardTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testWildcard() throws Exception { - index(client("1"), "1", "010"); - index(client("1"), "2", "0*0"); + void testWildcard() throws Exception { + ElasticsearchClient client = helper.client("1"); + index(client, "1", "010"); + index(client, "2", "0*0"); // exact - validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1); - validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("010").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1); // pattern - validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2? - validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2? - validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2? - validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0); - validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1); - validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1); - validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1? - validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1? + validateCount(client, QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2? + validateCount(client, QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2? + validateCount(client, QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2? + validateCount(client, QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0); + validateCount(client, QueryBuilders.queryStringQuery("*10").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1? + validateCount(client, QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1? } - private void index(Client client, String id, String fieldValue) throws IOException { + private void index(ElasticsearchClient client, String id, String fieldValue) throws IOException { client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id) .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())) .actionGet(); client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); } - private long count(Client client, QueryBuilder queryBuilder) { + private long count(ElasticsearchClient client, QueryBuilder queryBuilder) { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(queryBuilder); SearchRequest searchRequest = new SearchRequest(); @@ -52,7 +61,7 @@ public class WildcardTest extends TestBase { return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); } - private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { + private void validateCount(ElasticsearchClient client, QueryBuilder queryBuilder, long expectedHits) { final long actualHits = count(client, queryBuilder); if (actualHits != expectedHits) { throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java index c8faf35..0e8241b 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java +++ b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java @@ -15,7 +15,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.net.URL; import org.xbib.netty.http.client.Client; +import org.xbib.netty.http.client.ClientBuilder; import java.io.IOException; import java.util.ArrayList; @@ -59,14 +61,26 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti if (settings == null) { return null; } - this.url = settings.get("url"); + if (settings.hasValue("url")) { + this.url = settings.get("url"); + } else if (settings.hasValue("host")) { + this.url = URL.http() + .host(settings.get("host")).port(settings.getAsInt("port", 9200)) + .build() + .toExternalForm(); + } ServiceLoader httpActionServiceLoader = ServiceLoader.load(HttpAction.class, classLoader); for (HttpAction httpAction : httpActionServiceLoader) { httpAction.setSettings(settings); actionMap.put(httpAction.getActionInstance(), httpAction); } - this.nettyHttpClient = Client.builder().enableDebug().build(); - logger.info("extended HTTP client initialized with {} actions", actionMap.size()); + ClientBuilder clientBuilder = Client.builder(); + if (settings.hasValue("debug")) { + clientBuilder.enableDebug(); + } + this.nettyHttpClient = clientBuilder.build(); + logger.info("extended HTTP client initialized, settings = {}, url = {}, {} actions", + settings, url, actionMap.size()); return this; } @@ -102,7 +116,6 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti RequestBuilder extends ActionRequestBuilder> ActionFuture execute(Action action, Request request) { PlainActionFuture actionFuture = PlainActionFuture.newFuture(); - logger.info("plain action future = " + actionFuture); execute(action, request, actionFuture); return actionFuture; } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java index 920dd5e..b41c114 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java @@ -12,10 +12,11 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.http.ExtendedHttpClient; @@ -26,34 +27,31 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -@Ignore -public class ClientTest extends TestBase { +@Disabled +@ExtendWith(TestExtension.class) +class ClientTest { private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); - private static final Long ACTIONS = 25000L; + private static final Long ACTIONS = 100L; - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - @Before - public void startNodes() { - try { - super.startNodes(); - startNode("2"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } + private final TestExtension.Helper helper; + + ClientTest(TestExtension.Helper helper) { + this.helper = helper; } @Test - public void testSingleDoc() throws Exception { + void testSingleDoc() throws Exception { final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) .build(); @@ -75,9 +73,10 @@ public class ClientTest extends TestBase { } @Test - public void testNewIndex() throws Exception { + void testNewIndex() throws Exception { final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); client.newIndex("test"); @@ -85,12 +84,13 @@ public class ClientTest extends TestBase { } @Test - public void testMapping() throws Exception { + void testMapping() throws Exception { final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); - XContentBuilder builder = jsonBuilder() + XContentBuilder builder = JsonXContent.contentBuilder() .startObject() .startObject("doc") .startObject("properties") @@ -110,17 +110,18 @@ public class ClientTest extends TestBase { } @Test - public void testRandomDocs() throws Exception { + void testRandomDocs() throws Exception { long numactions = ACTIONS; final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); @@ -144,13 +145,14 @@ public class ClientTest extends TestBase { } @Test - public void testThreadedRandomDocs() throws Exception { + void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; final Long actions = ACTIONS; logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) @@ -168,7 +170,7 @@ public class ClientTest extends TestBase { for (int i = 0; i < maxthreads; i++) { executorService.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java index 65745ce..c5aa731 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java @@ -5,8 +5,9 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.http.ExtendedHttpClient; @@ -15,12 +16,13 @@ import org.xbib.elx.http.ExtendedHttpClientProvider; import java.util.concurrent.TimeUnit; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -@Ignore -public class DuplicateIDTest extends TestBase { +@Disabled +@ExtendWith(TestExtension.class) +class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); @@ -28,17 +30,25 @@ public class DuplicateIDTest extends TestBase { private static final Long ACTIONS = 12345L; + private final TestExtension.Helper helper; + + DuplicateIDTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testDuplicateDocIDs() throws Exception { + void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java new file mode 100644 index 0000000..8873235 --- /dev/null +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java @@ -0,0 +1,88 @@ +package org.xbib.elx.http.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.api.IndexPruneResult; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.http.ExtendedHttpClient; +import org.xbib.elx.http.ExtendedHttpClientProvider; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Disabled +@ExtendWith(TestExtension.class) +class IndexPruneTest { + + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + + private final TestExtension.Helper helper; + + IndexPruneTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testPrune() throws IOException { + final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1")) + .put(helper.getHttpSettings()) + .provider(ExtendedHttpClientProvider.class) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + client.newIndex("test1", settings); + client.shiftIndex("test", "test1", Collections.emptyList()); + client.newIndex("test2", settings); + client.shiftIndex("test", "test2", Collections.emptyList()); + client.newIndex("test3", settings); + client.shiftIndex("test", "test3", Collections.emptyList()); + client.newIndex("test4", settings); + client.shiftIndex("test", "test4", Collections.emptyList()); + IndexPruneResult indexPruneResult = + client.pruneIndex("test", "test4", 2, 2, true); + assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); + assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); + List list = new ArrayList<>(); + for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { + IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); + indicesExistsRequest.indices(index); + IndicesExistsResponse indicesExistsResponse = + client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + list.add(indicesExistsResponse.isExists()); + } + logger.info(list); + assertFalse(list.get(0)); + assertFalse(list.get(1)); + assertTrue(list.get(2)); + assertTrue(list.get(3)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + } + } +} diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java index 7317978..ea0f8e9 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java @@ -6,29 +6,37 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.http.ExtendedHttpClient; import org.xbib.elx.http.ExtendedHttpClientProvider; import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -@Ignore -public class IndexShiftTest extends TestBase { +@Disabled +@ExtendWith(TestExtension.class) +class IndexShiftTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); + private final TestExtension.Helper helper; + + IndexShiftTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testIndexShift() throws Exception { + void testIndexShift() throws Exception { final ExtendedHttpClient client = ClientBuilder.builder() + .put(helper.getHttpSettings()) .provider(ExtendedHttpClientProvider.class) .build(); try { @@ -38,7 +46,8 @@ public class IndexShiftTest extends TestBase { .build(); client.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); @@ -66,7 +75,8 @@ public class IndexShiftTest extends TestBase { client.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ReplicaTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/ReplicaTest.java deleted file mode 100644 index c9037ca..0000000 --- a/elx-http/src/test/java/org/xbib/elx/http/test/ReplicaTest.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.xbib.elx.http.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.shard.IndexingStats; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -@Ignore -public class ReplicaTest extends TestBase { - - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); - - @Test - public void testReplicaLevel() throws Exception { - - // we need nodes for replica levels - startNode("2"); - startNode("3"); - startNode("4"); - - Settings settingsTest1 = Settings.builder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 3) - .build(); - - Settings settingsTest2 = Settings.builder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 1) - .build(); - - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .build(); - - try { - client.newIndex("test1", settingsTest1, new HashMap<>()) - .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 1234; i++) { - client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - for (int i = 0; i < 1234; i++) { - client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - logger.info("refreshing"); - client.refreshIndex("test1"); - client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test1", "test2") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("query total hits={}", hits); - assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE) - .all(); - IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); - for (Map.Entry m : response.getIndices().entrySet()) { - IndexStats indexStats = m.getValue(); - CommonStats commonStats = indexStats.getTotal(); - IndexingStats indexingStats = commonStats.getIndexing(); - IndexingStats.Stats stats = indexingStats.getTotal(); - logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); - for (Map.Entry me : indexStats.getIndexShards().entrySet()) { - IndexShardStats indexShardStats = me.getValue(); - CommonStats commonShardStats = indexShardStats.getTotal(); - logger.info("shard {} count = {}", me.getKey(), - commonShardStats.getIndexing().getTotal().getIndexCount()); - } - } - try { - client.deleteIndex("test1") - .deleteIndex("test2"); - } catch (Exception e) { - logger.error("delete index failed, ignored. Reason:", e); - } - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - Settings settings = Settings.builder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 12345; i++) { - client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); - assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - -} diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java index 30dc44a..dfcaf50 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java @@ -4,8 +4,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.http.ExtendedHttpClient; @@ -13,37 +14,39 @@ import org.xbib.elx.http.ExtendedHttpClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -@Ignore -public class SmokeTest extends TestBase { +@Disabled +@ExtendWith(TestExtension.class) +class SmokeTest { private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); + private final TestExtension.Helper helper; + + SmokeTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void smokeTest() throws Exception { + void smokeTest() throws Exception { final ExtendedHttpClient client = ClientBuilder.builder() .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .build(); try { client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - - assertEquals(getClusterName(), client.getClusterName()); - + assertEquals(helper.getClusterName(), client.getClusterName()); client.checkMapping("test"); - client.update("test", "1", "{ \"name\" : \"Another name\"}"); client.flush(); - client.waitForRecovery("test", 10L, TimeUnit.SECONDS); - client.delete("test", "1"); client.deleteIndex("test"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); @@ -51,10 +54,8 @@ public class SmokeTest extends TestBase { client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.flush(); client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java b/elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java deleted file mode 100644 index d15a279..0000000 --- a/elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java +++ /dev/null @@ -1,198 +0,0 @@ -package org.xbib.elx.http.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; -import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeValidationException; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; - -public class TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - private static final Random random = new Random(); - - private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private AtomicInteger counter = new AtomicInteger(); - - private String cluster; - - private String host; - - private int port; - - @Before - public void startNodes() { - try { - logger.info("starting"); - this.cluster = "test-helper-cluster-" + counter.incrementAndGet(); - startNode("1"); - findNodeAddress(); - try { - ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) - .timeout(TimeValue.timeValueSeconds(30))).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); - } - } catch (Throwable t) { - logger.error("startNodes failed", t); - } - } - - @After - public void stopNodes() { - try { - closeNodes(); - } catch (Exception e) { - logger.error("can not close nodes", e); - } finally { - try { - deleteFiles(); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected Settings getTransportSettings() { - return Settings.builder() - .put("host", host) - .put("port", port) - .put("cluster.name", cluster) - .put("path.home", getHome()) - .build(); - } - - protected Settings getNodeSettings() { - return Settings.builder() - .put("cluster.name", cluster) - .put("discovery.zen.minimum_master_nodes", "1") - .put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here - .put("path.home", getHome()) - .build(); - } - - protected static String getHome() { - return System.getProperty("path.home", System.getProperty("user.dir")); - } - - protected void startNode(String id) throws NodeValidationException { - buildNode(id).start(); - } - - protected AbstractClient client(String id) { - return clients.get(id); - } - - protected String getClusterName() { - return cluster; - } - - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - private void closeNodes() throws IOException { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - - private void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress() - .publishAddress(); - host = address.address().getHostName(); - port = address.address().getPort(); - } - - private Node buildNode(String id) { - Settings nodeSettings = Settings.builder() - .put(getNodeSettings()) - .put("node.name", id) - .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); - Node node = new MockNode(nodeSettings, plugins); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - return node; - } - - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } -} diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java b/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java new file mode 100644 index 0000000..4255c08 --- /dev/null +++ b/elx-http/src/test/java/org/xbib/elx/http/test/TestExtension.java @@ -0,0 +1,236 @@ +package org.xbib.elx.http.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.analysis.common.CommonAnalysisPlugin; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Junit 5 extension for testing Elasticsearch. + * The extension will be instantiated as a singleton. + * For parallel test method executions, for example in gradle, it requires a helper class + * to ensure different ES homes/clusters for each run. + */ +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private static final String key = "es-instance-"; + + private static final AtomicInteger count = new AtomicInteger(0); + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + TransportAddress address = response.getNodes().get(0).getHttp().getAddress().publishAddress(); + helper.httpHost = address.address().getHostName(); + helper.httpPort = address.address().getPort(); + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome())); + logger.info("data files wiped: " + helper.getHome()); + Thread.sleep(2000L); // let OS commit changes + } + + private void closeNodes(Helper helper) throws IOException { + logger.info("closing all clients"); + for (AbstractClient client : helper.clients.values()) { + client.close(); + } + logger.info("closing all nodes"); + for (Node node : helper.nodes.values()) { + if (node != null) { + node.close(); + } + } + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; + } + + class Helper { + + String home; + + String cluster; + + String httpHost; + + int httpPort; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } + + Settings getNodeSettings() { + return Settings.builder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } + + Settings getHttpSettings() { + return Settings.builder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .put("host", httpHost) + .put("port", httpPort) + .build(); + } + + void startNode(String id) throws NodeValidationException { + buildNode(id).start(); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + private Node buildNode(String id) { + Settings nodeSettings = Settings.builder() + .put(getNodeSettings()) + .put("node.name", id) + .build(); + List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + Node node = new MockNode(nodeSettings, plugins); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + return node; + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java index 9673e81..0cc3b10 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java @@ -1,10 +1,5 @@ package org.xbib.elx.node.test; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; @@ -17,9 +12,10 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.node.ExtendedNodeClient; @@ -30,27 +26,28 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -public class ClientTest extends TestBase { +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class ClientTest { private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - private static final Long ACTIONS = 25000L; + private static final Long ACTIONS = 1000L; - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 100L; - @Before - public void startNodes() { - try { - super.startNodes(); - startNode("2"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } + private final TestExtension.Helper helper; + + ClientTest(TestExtension.Helper helper) { + this.helper = helper; } @Test - public void testSingleDoc() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testSingleDoc() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) @@ -73,8 +70,8 @@ public class ClientTest extends TestBase { } @Test - public void testNewIndex() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testNewIndex() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); @@ -83,12 +80,12 @@ public class ClientTest extends TestBase { } @Test - public void testMapping() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testMapping() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); - XContentBuilder builder = jsonBuilder() + XContentBuilder builder = JsonXContent.contentBuilder() .startObject() .startObject("doc") .startObject("properties") @@ -108,9 +105,9 @@ public class ClientTest extends TestBase { } @Test - public void testRandomDocs() throws Exception { + void testRandomDocs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) @@ -118,7 +115,8 @@ public class ClientTest extends TestBase { try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); @@ -140,12 +138,12 @@ public class ClientTest extends TestBase { } @Test - public void testThreadedRandomDocs() throws Exception { + void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; final Long actions = ACTIONS; logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) @@ -164,7 +162,8 @@ public class ClientTest extends TestBase { for (int i = 0; i < maxthreads; i++) { executorService.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index ed84a17..4fa3869 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -7,7 +7,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.node.ExtendedNodeClient; @@ -15,28 +16,37 @@ import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class DuplicateIDTest extends TestBase { +@ExtendWith(TestExtension.class) +class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 12345L; + private static final Long ACTIONS = 5L; + + private final TestExtension.Helper helper; + + DuplicateIDTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testDuplicateDocIDs() throws Exception { + void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); @@ -47,7 +57,7 @@ public class DuplicateIDTest extends TestBase { searchRequest.indices("test"); searchRequest.types("test"); searchRequest.source(builder); - long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); + long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 8b5afb4..b729716 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.node.ExtendedNodeClient; @@ -19,17 +20,24 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexPruneTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexPruneTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexPruneTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testPrune() throws IOException { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testPrune() throws IOException { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .build(); try { @@ -45,15 +53,12 @@ public class IndexPruneTest extends TestBase { client.shiftIndex("test", "test3", Collections.emptyList()); client.newIndex("test4", settings); client.shiftIndex("test", "test4", Collections.emptyList()); - IndexPruneResult indexPruneResult = client.pruneIndex("test", "test4", 2, 2, true); - assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); - List list = new ArrayList<>(); for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index 3bacae7..3ca8c69 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -6,7 +6,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.node.ExtendedNodeClient; @@ -16,16 +17,23 @@ import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexShiftTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexShiftTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexShiftTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testIndexShift() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testIndexShift() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .build(); try { @@ -35,14 +43,13 @@ public class IndexShiftTest extends TestBase { .build(); client.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); - assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -63,7 +70,8 @@ public class IndexShiftTest extends TestBase { client.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java deleted file mode 100644 index 66fb64d..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.xbib.elx.node.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.shard.IndexingStats; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -@Ignore -public class ReplicaTest extends TestBase { - - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName()); - - @Test - public void testReplicaLevel() throws Exception { - - // we need nodes for replica levels - startNode("2"); - startNode("3"); - startNode("4"); - - Settings settingsTest1 = Settings.builder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 3) - .build(); - - Settings settingsTest2 = Settings.builder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 1) - .build(); - - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - - try { - client.newIndex("test1", settingsTest1, new HashMap<>()) - .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 1234; i++) { - client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - for (int i = 0; i < 1234; i++) { - client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - logger.info("refreshing"); - client.refreshIndex("test1"); - client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test1", "test2") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("query total hits={}", hits); - assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE) - .all(); - IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); - for (Map.Entry m : response.getIndices().entrySet()) { - IndexStats indexStats = m.getValue(); - CommonStats commonStats = indexStats.getTotal(); - IndexingStats indexingStats = commonStats.getIndexing(); - IndexingStats.Stats stats = indexingStats.getTotal(); - logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); - for (Map.Entry me : indexStats.getIndexShards().entrySet()) { - IndexShardStats indexShardStats = me.getValue(); - CommonStats commonShardStats = indexShardStats.getTotal(); - logger.info("shard {} count = {}", me.getKey(), - commonShardStats.getIndexing().getTotal().getIndexCount()); - } - } - try { - client.deleteIndex("test1") - .deleteIndex("test2"); - } catch (Exception e) { - logger.error("delete index failed, ignored. Reason:", e); - } - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - Settings settings = Settings.builder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 12345; i++) { - client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); - assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index b675e3a..a4b4209 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.node.ExtendedNodeClient; @@ -12,31 +13,35 @@ import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -public class SmokeTest extends TestBase { +@ExtendWith(TestExtension.class) +class SmokeTest { private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); + private final TestExtension.Helper helper; + + SmokeTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void smokeTest() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void smokeTest() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .build(); try { - assertEquals(getClusterName(), client.getClusterName()); - + assertEquals(helper.getClusterName(), client.getClusterName()); client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.update("test", "1", "{ \"name\" : \"Another name\"}"); client.delete("test", "1"); client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test"); client.deleteIndex("test"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); @@ -46,10 +51,8 @@ public class SmokeTest extends TestBase { client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java deleted file mode 100644 index c511fb3..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.xbib.elx.node.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; -import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeValidationException; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -public class TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - private static final Random random = new Random(); - - private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String cluster; - - private String host; - - private int port; - - @Before - public void startNodes() { - try { - logger.info("starting"); - this.cluster = "test-cluster-" + System.getProperty("user.name"); - startNode("1"); - findNodeAddress(); - try { - ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) - .timeout(TimeValue.timeValueSeconds(30))).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); - } - ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.all(); - ClusterStateResponse clusterStateResponse = - client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); - logger.info("host = {} port = {}", host, port); - } catch (Throwable t) { - logger.error(t.getMessage(), t); - } - } - - @After - public void stopNodes() { - try { - closeNodes(); - } catch (Exception e) { - logger.error("can not close nodes", e); - } finally { - try { - deleteFiles(); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected Settings getTransportSettings() { - return Settings.builder() - .put("host", host) - .put("port", port) - .put("cluster.name", cluster) - .put("path.home", getHome()) - .build(); - } - - protected Settings getNodeSettings() { - return Settings.builder() - .put("cluster.name", cluster) - .put("discovery.zen.minimum_master_nodes", "1") - .put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here - .put("path.home", getHome()) - .build(); - } - - protected static String getHome() { - return System.getProperty("path.home", System.getProperty("user.dir")); - } - - protected void startNode(String id) throws NodeValidationException { - buildNode(id).start(); - } - - protected AbstractClient client(String id) { - return clients.get(id); - } - - protected String getClusterName() { - return cluster; - } - - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - private void closeNodes() throws IOException { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - - private void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress() - .publishAddress(); - host = address.address().getHostName(); - port = address.address().getPort(); - } - - private Node buildNode(String id) { - Settings nodeSettings = Settings.builder() - .put(getNodeSettings()) - .put("node.name", id) - .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); - Node node = new MockNode(nodeSettings, plugins); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - return node; - } - - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java new file mode 100644 index 0000000..06d8893 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java @@ -0,0 +1,236 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.analysis.common.CommonAnalysisPlugin; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Junit 5 extension for testing Elasticsearch. + * The extension will be instantiated as a singleton. + * For parallel test method executions, for example in gradle, it requires a helper class + * to ensure different ES homes/clusters for each run. + */ +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private static final String key = "es-instance-"; + + private static final AtomicInteger count = new AtomicInteger(0); + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress(); + helper.host = address.address().getHostName(); + helper.port = address.address().getPort(); + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome())); + logger.info("data files wiped: " + helper.getHome()); + Thread.sleep(2000L); // let OS commit changes + } + + private void closeNodes(Helper helper) throws IOException { + logger.info("closing all clients"); + for (AbstractClient client : helper.clients.values()) { + client.close(); + } + logger.info("closing all nodes"); + for (Node node : helper.nodes.values()) { + if (node != null) { + node.close(); + } + } + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; + } + + class Helper { + + String home; + + String cluster; + + String host; + + int port; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } + + Settings getNodeSettings() { + return Settings.builder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } + + Settings getTransportSettings() { + return Settings.builder() + .put("cluster.name", cluster) + .put("path.home", getHome()) + .put("host", host) + .put("port", port) + .build(); + } + + void startNode(String id) throws NodeValidationException { + buildNode(id).start(); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + private Node buildNode(String id) { + Settings nodeSettings = Settings.builder() + .put(getNodeSettings()) + .put("node.name", id) + .build(); + List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + Node node = new MockNode(nodeSettings, plugins); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + return node; + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java index 3518256..0300f96 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java @@ -13,10 +13,11 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.transport.ExtendedTransportClient; @@ -27,36 +28,32 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class ClientTest extends TestBase { +@ExtendWith(TestExtension.class) +class ClientTest { private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - private static final Long ACTIONS = 25000L; + private static final Long ACTIONS = 1000L; - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 100L; - @Before - public void startNodes() { - try { - super.startNodes(); - startNode("2"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } + private final TestExtension.Helper helper; + + ClientTest(TestExtension.Helper helper) { + this.helper = helper; } @Test - public void testSingleDoc() throws Exception { + void testSingleDoc() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { client.newIndex("test"); @@ -76,24 +73,24 @@ public class ClientTest extends TestBase { } @Test - public void testNewIndex() throws Exception { + void testNewIndex() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); client.newIndex("test"); client.close(); } @Test - public void testMapping() throws Exception { + void testMapping() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); - XContentBuilder builder = jsonBuilder() + XContentBuilder builder = JsonXContent.contentBuilder() .startObject() .startObject("doc") .startObject("properties") @@ -113,18 +110,19 @@ public class ClientTest extends TestBase { } @Test - public void testRandomDocs() throws Exception { + void testRandomDocs() throws Exception { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); @@ -152,7 +150,7 @@ public class ClientTest extends TestBase { } @Test - public void testThreadedRandomDocs() throws Exception { + void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; final Long actions = ACTIONS; @@ -162,7 +160,7 @@ public class ClientTest extends TestBase { .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { Settings settings = Settings.builder() @@ -177,7 +175,8 @@ public class ClientTest extends TestBase { for (int i = 0; i < maxthreads; i++) { executorService.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index 99b93cb..fe98d3c 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -8,7 +8,8 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.transport.ExtendedTransportClient; @@ -16,30 +17,38 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class DuplicateIDTest extends TestBase { +@ExtendWith(TestExtension.class) +class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 12345L; + private static final Long ACTIONS = 5L; + + private final TestExtension.Helper helper; + + DuplicateIDTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testDuplicateDocIDs() throws Exception { + void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); @@ -52,7 +61,7 @@ public class DuplicateIDTest extends TestBase { searchRequest.types("test"); searchRequest.source(builder); SearchResponse searchResponse = - client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); + helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); long hits = searchResponse.getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 9f0b89f..c2b46bc 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.transport.ExtendedTransportClient; @@ -19,19 +20,26 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexPruneTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexPruneTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexPruneTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testPrune() throws IOException { + void testPrune() throws IOException { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { Settings settings = Settings.builder() @@ -46,15 +54,12 @@ public class IndexPruneTest extends TestBase { client.shiftIndex("test", "test3", Collections.emptyList()); client.newIndex("test4", settings); client.shiftIndex("test", "test4", Collections.emptyList()); - IndexPruneResult indexPruneResult = client.pruneIndex("test", "test4", 2, 2, true); - assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); - List list = new ArrayList<>(); for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 1f178ed..a26f4d9 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -6,31 +6,36 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Ignore; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.transport.ExtendedTransportClient; import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -@Ignore -public class IndexShiftTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexShiftTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexShiftTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testIndexShift() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder(client("1")) + void testIndexShift() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { Settings settings = Settings.builder() @@ -39,39 +44,35 @@ public class IndexShiftTest extends TestBase { .build(); client.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); - assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - Map aliases = client.getAliases("test1234"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - String resolved = client.resolveAlias("test"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - client.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) @@ -82,7 +83,6 @@ public class IndexShiftTest extends TestBase { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - aliases = client.getAliases("test5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); @@ -90,7 +90,6 @@ public class IndexShiftTest extends TestBase { assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - resolved = client.resolveAlias("test"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); @@ -99,7 +98,6 @@ public class IndexShiftTest extends TestBase { assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ReplicaTest.java deleted file mode 100644 index d606f67..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ReplicaTest.java +++ /dev/null @@ -1,161 +0,0 @@ -package org.xbib.elx.transport.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.shard.IndexingStats; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -@Ignore -public class ReplicaTest extends TestBase { - - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName()); - - @Test - public void testReplicaLevel() throws Exception { - - // we need nodes for replica levels - startNode("2"); - startNode("3"); - startNode("4"); - - Settings settingsTest1 = Settings.builder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 3) - .build(); - - Settings settingsTest2 = Settings.builder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 1) - .build(); - - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) - .build(); - - try { - client.newIndex("test1", settingsTest1, new HashMap<>()) - .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 1234; i++) { - client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - for (int i = 0; i < 1234; i++) { - client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - logger.info("refreshing"); - client.refreshIndex("test1"); - client.refreshIndex("test2"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test1", "test2"); - searchRequest.source(builder); - SearchResponse searchResponse = - client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits(); - logger.info("query total hits={}", hits); - assertEquals(2468, hits); - IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); - indicesStatsRequest.all(); - IndicesStatsResponse response = - client.getClient().execute(IndicesStatsAction.INSTANCE, indicesStatsRequest).actionGet(); - for (Map.Entry m : response.getIndices().entrySet()) { - IndexStats indexStats = m.getValue(); - CommonStats commonStats = indexStats.getTotal(); - IndexingStats indexingStats = commonStats.getIndexing(); - IndexingStats.Stats stats = indexingStats.getTotal(); - logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); - for (Map.Entry me : indexStats.getIndexShards().entrySet()) { - IndexShardStats indexShardStats = me.getValue(); - CommonStats commonShardStats = indexShardStats.getTotal(); - logger.info("shard {} count = {}", me.getKey(), - commonShardStats.getIndexing().getTotal().getIndexCount()); - } - } - try { - client.deleteIndex("test1") - .deleteIndex("test2"); - } catch (Exception e) { - logger.error("delete index failed, ignored. Reason:", e); - } - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - Settings settings = Settings.builder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 12345; i++) { - client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); - assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index 27fc5e4..0435439 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.transport.ExtendedTransportClient; @@ -12,22 +13,28 @@ import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -public class SmokeTest extends TestBase { +@ExtendWith(TestExtension.class) +class SmokeTest { private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); - @Test - public void smokeTest() throws Exception { + private final TestExtension.Helper helper; + + SmokeTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test + void smokeTest() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { - assertEquals(getClusterName(), client.getClusterName()); + assertEquals(helper.getClusterName(), client.getClusterName()); client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.update("test", "1", "{ \"name\" : \"Another name\"}"); @@ -36,7 +43,6 @@ public class SmokeTest extends TestBase { client.waitForResponses(30, TimeUnit.SECONDS); client.checkMapping("test"); client.deleteIndex("test"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test", Settings.builder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); @@ -47,7 +53,6 @@ public class SmokeTest extends TestBase { client.updateReplicaLevel(indexDefinition, 2); int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestBase.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestBase.java deleted file mode 100644 index 957aced..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestBase.java +++ /dev/null @@ -1,204 +0,0 @@ -package org.xbib.elx.transport.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.analysis.common.CommonAnalysisPlugin; -import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.elasticsearch.node.NodeValidationException; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -public class TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - private static final Random random = new Random(); - - private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String cluster; - - private String host; - - private int port; - - @Before - public void startNodes() { - try { - logger.info("starting"); - this.cluster = "test-cluster-" + System.getProperty("user.name"); - startNode("1"); - findNodeAddress(); - try { - ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) - .timeout(TimeValue.timeValueSeconds(30))).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); - } - ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); - clusterStateRequest.all(); - ClusterStateResponse clusterStateResponse = - client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); - logger.info("host = {} port = {}", host, port); - } catch (Throwable t) { - logger.error(t.getMessage(), t); - } - } - - @After - public void stopNodes() { - try { - closeNodes(); - } catch (Exception e) { - logger.error("can not close nodes", e); - } finally { - try { - deleteFiles(); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected Settings getTransportSettings() { - return Settings.builder() - .put("cluster.name", cluster) - .put("path.home", getHome()) - .put("host", host) - .put("port", port) - .build(); - } - - protected Settings getNodeSettings() { - return Settings.builder() - .put("cluster.name", cluster) - .put("discovery.zen.minimum_master_nodes", "1") - .put("transport.type", Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("node.max_local_storage_nodes", 10) // allow many nodes to initialize here - .put("path.home", getHome()) - .build(); - } - - protected static String getHome() { - return System.getProperty("path.home", System.getProperty("user.dir")); - } - - protected void startNode(String id) throws NodeValidationException { - buildNode(id).start(); - } - - protected AbstractClient client(String id) { - return clients.get(id); - } - - protected String getClusterName() { - return cluster; - } - - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - private void closeNodes() throws IOException { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - - private void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - TransportAddress address= response.getNodes().iterator().next().getTransport().getAddress() - .publishAddress(); - host = address.address().getHostName(); - port = address.address().getPort(); - } - - private Node buildNode(String id) { - Settings nodeSettings = Settings.builder() - .put(getNodeSettings()) - .put("node.name", id) - .build(); - List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); - Node node = new MockNode(nodeSettings, plugins); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - return node; - } - - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor<>() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java new file mode 100644 index 0000000..a6a285f --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java @@ -0,0 +1,236 @@ +package org.xbib.elx.transport.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.analysis.common.CommonAnalysisPlugin; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Junit 5 extension for testing Elasticsearch. + * The extension will be instantiated as a singleton. + * For parallel test method executions, for example in gradle, it requires a helper class + * to ensure different ES homes/clusters for each run. + */ +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private static final String key = "es-instance-"; + + private static final AtomicInteger count = new AtomicInteger(0); + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + TransportAddress address = response.getNodes().get(0).getTransport().getAddress().publishAddress(); + helper.host = address.address().getHostName(); + helper.port = address.address().getPort(); + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + } + + @Override + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome())); + logger.info("data files wiped: " + helper.getHome()); + Thread.sleep(2000L); // let OS commit changes + } + + private void closeNodes(Helper helper) throws IOException { + logger.info("closing all clients"); + for (AbstractClient client : helper.clients.values()) { + client.close(); + } + logger.info("closing all nodes"); + for (Node node : helper.nodes.values()) { + if (node != null) { + node.close(); + } + } + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; + } + + class Helper { + + String home; + + String cluster; + + String host; + + int port; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } + + Settings getNodeSettings() { + return Settings.builder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } + + Settings getTransportSettings() { + return Settings.builder() + .put("cluster.name", cluster) + .put("path.home", getHome()) + .put("host", host) + .put("port", port) + .build(); + } + + void startNode(String id) throws NodeValidationException { + buildNode(id).start(); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + private Node buildNode(String id) { + Settings nodeSettings = Settings.builder() + .put(getNodeSettings()) + .put("node.name", id) + .build(); + List> plugins = Arrays.asList(CommonAnalysisPlugin.class, Netty4Plugin.class); + Node node = new MockNode(nodeSettings, plugins); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + return node; + } + } +} diff --git a/gradle.properties b/gradle.properties index aff5d32..832cfd3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,17 +1,17 @@ group = org.xbib name = elx -version = 6.3.2.1 +version = 6.3.2.2 profile = default release = 0 -elasticsearch-server.version = 6.3.2.2 +elasticsearch-server.version = 6.3.2.3 log4j.version = 2.11.1 xbib-metrics.version = 1.2.0 -xbib-netty-http.version = 4.1.33.0 +xbib-netty-http.version = 4.1.35.0 # test -junit.version = 4.12 -wagon.version = 3.0.0 +junit.version = 5.4.2 +wagon.version = 3.3.2 asciidoclet.version = 1.6.0.0 org.gradle.warning.mode = all diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 87b738c..5c2d1cf 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 0bcaeeb..838e6bc 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Fri Feb 15 10:39:20 CET 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip diff --git a/gradlew b/gradlew index af6708f..b0d6d0a 100755 --- a/gradlew +++ b/gradlew @@ -1,5 +1,21 @@ #!/usr/bin/env sh +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + ############################################################################## ## ## Gradle start up script for UN*X @@ -28,7 +44,7 @@ APP_NAME="Gradle" APP_BASE_NAME=`basename "$0"` # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m"' +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD="maximum" diff --git a/gradlew.bat b/gradlew.bat index 0f8d593..15e1ee3 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -1,3 +1,19 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + @if "%DEBUG%" == "" @echo off @rem ########################################################################## @rem @@ -14,7 +30,7 @@ set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @rem Find java.exe if defined JAVA_HOME goto findJavaFromJavaHome