From fcd99f801df91765905971d7eb6b60e66e9b5cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Fri, 22 Feb 2019 20:40:29 +0100 Subject: [PATCH 01/19] changes from 6.3 branch, some fixes, better timeouts, better index shift --- elx-api/build.gradle~ | 18 -- .../java/org/xbib/elx/api/BulkProcessor.java | 6 +- .../java/org/xbib/elx/api/ExtendedClient.java | 20 +- .../org/xbib/elx/api/IndexAliasAdder.java | 4 +- .../java/org/xbib/elx/api/package-info.java | 2 +- elx-common/build.gradle | 2 - elx-common/build.gradle~ | 65 ----- .../elx/common/AbstractExtendedClient.java | 231 +++++++++++------- .../elx/common/DefaultBulkController.java | 9 +- .../xbib/elx/common/DefaultBulkProcessor.java | 8 +- .../xbib/elx/common/MockExtendedClient.java | 7 +- .../java/org/elasticsearch/node/MockNode.java | 34 --- .../org/elasticsearch/node/package-info.java | 1 - .../java/org/xbib/elx/common/SimpleTest.java | 57 ----- .../org/xbib/elx/common/package-info.java | 1 - .../xbib/elx/common/{ => test}/AliasTest.java | 11 +- .../elx/common/test}/ClusterBlockTest.java | 7 +- .../MockExtendedClientProviderTest.java | 5 +- .../org/xbib/elx/common/test/MockNode.java | 12 + .../elx/common/{ => test}/NetworkTest.java | 4 +- .../elx/common/{ => test}/SearchTest.java | 7 +- .../org/xbib/elx/common/test/SimpleTest.java | 61 +++++ .../org/xbib/elx/common/test/TestBase.java | 158 +++++++----- .../elx/common/{ => test}/WildcardTest.java | 31 ++- .../xbib/elx/common/test/package-info.java | 1 + elx-http/build.gradle~ | 65 ----- elx-node/build.gradle~ | 65 ----- .../org/xbib/elx/node/ExtendedNodeClient.java | 15 +- .../java/org/elasticsearch/node/MockNode.java | 30 --- .../org/xbib/elx/node/IndexShiftTest.java | 77 ------ .../xbib/elx/node/{ => test}/ClientTest.java | 10 +- .../elx/node/{ => test}/DuplicateIDTest.java | 26 +- .../xbib/elx/node/test/IndexShiftTest.java | 111 +++++++++ .../java/org/xbib/elx/node/test/MockNode.java | 12 + .../xbib/elx/node/{ => test}/ReplicaTest.java | 8 +- .../xbib/elx/node/{ => test}/SmokeTest.java | 10 +- .../java/org/xbib/elx/node/test/TestBase.java | 121 +++++---- elx-transport/build.gradle~ | 63 ----- .../transport/ExtendedTransportClient.java | 21 +- .../java/org/elasticsearch/node/MockNode.java | 34 --- .../org/elasticsearch/node/package-info.java | 1 - .../org/xbib/elx/transport/ClientTest.java | 14 +- .../xbib/elx/transport/DuplicateIDTest.java | 25 +- .../xbib/elx/transport/IndexShiftTest.java | 79 ++++-- .../java/org/xbib/elx/transport/MockNode.java | 11 + .../org/xbib/elx/transport/ReplicaTest.java | 8 +- .../org/xbib/elx/transport/SmokeTest.java | 36 ++- .../{NodeTestUtils.java => TestBase.java} | 120 +++++---- elx-transport/src/test/resources/log4j2.xml | 2 +- gradle.properties | 2 +- gradle/publish.gradle~ | 104 -------- 51 files changed, 795 insertions(+), 1037 deletions(-) delete mode 100644 elx-api/build.gradle~ delete mode 100644 elx-common/build.gradle~ delete mode 100644 elx-common/src/test/java/org/elasticsearch/node/MockNode.java delete mode 100644 elx-common/src/test/java/org/elasticsearch/node/package-info.java delete mode 100644 elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java delete mode 100644 elx-common/src/test/java/org/xbib/elx/common/package-info.java rename elx-common/src/test/java/org/xbib/elx/common/{ => test}/AliasTest.java (90%) rename {elx-node/src/test/java/org/xbib/elx/node => elx-common/src/test/java/org/xbib/elx/common/test}/ClusterBlockTest.java (90%) rename elx-common/src/test/java/org/xbib/elx/common/{ => test}/MockExtendedClientProviderTest.java (67%) create mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java rename elx-common/src/test/java/org/xbib/elx/common/{ => test}/NetworkTest.java (93%) rename elx-common/src/test/java/org/xbib/elx/common/{ => test}/SearchTest.java (93%) create mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java rename elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java => elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java (65%) rename elx-common/src/test/java/org/xbib/elx/common/{ => test}/WildcardTest.java (69%) create mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/package-info.java delete mode 100644 elx-http/build.gradle~ delete mode 100644 elx-node/build.gradle~ delete mode 100644 elx-node/src/test/java/org/elasticsearch/node/MockNode.java delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java rename elx-node/src/test/java/org/xbib/elx/node/{ => test}/ClientTest.java (97%) rename elx-node/src/test/java/org/xbib/elx/node/{ => test}/DuplicateIDTest.java (69%) create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java rename elx-node/src/test/java/org/xbib/elx/node/{ => test}/ReplicaTest.java (96%) rename elx-node/src/test/java/org/xbib/elx/node/{ => test}/SmokeTest.java (90%) rename elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java => elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java (78%) delete mode 100644 elx-transport/build.gradle~ delete mode 100644 elx-transport/src/test/java/org/elasticsearch/node/MockNode.java delete mode 100644 elx-transport/src/test/java/org/elasticsearch/node/package-info.java create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java rename elx-transport/src/test/java/org/xbib/elx/transport/{NodeTestUtils.java => TestBase.java} (78%) delete mode 100644 gradle/publish.gradle~ diff --git a/elx-api/build.gradle~ b/elx-api/build.gradle~ deleted file mode 100644 index 02e43a4..0000000 --- a/elx-api/build.gradle~ +++ /dev/null @@ -1,18 +0,0 @@ - -dependencies { - compile("org.elasticsearch.client:transport:${rootProject.property('elasticsearch.version')}") { - exclude group: 'org.elasticsearch', module: 'securesm' - exclude group: 'org.elasticsearch.plugin', module: 'transport-netty3-client' - exclude group: 'org.elasticsearch.plugin', module: 'reindex-client' - exclude group: 'org.elasticsearch.plugin', module: 'percolator-client' - exclude group: 'org.elasticsearch.plugin', module: 'lang-mustache-client' - } - // we try to override the Elasticsearch netty by our netty version which might be more recent - compile "io.netty:netty-buffer:${rootProject.property('netty.version')}" - compile "io.netty:netty-codec-http:${rootProject.property('netty.version')}" - compile "io.netty:netty-handler:${rootProject.property('netty.version')}" -} - -jar { - baseName "${rootProject.name}-api" -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java index 5af92e1..2703acb 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -10,9 +10,11 @@ import java.util.concurrent.TimeUnit; public interface BulkProcessor extends Closeable, Flushable { - BulkProcessor add(ActionRequest request); + @SuppressWarnings("rawtype") + BulkProcessor add(ActionRequest request); - BulkProcessor add(ActionRequest request, Object payload); + @SuppressWarnings("rawtype") + BulkProcessor add(ActionRequest request, Object payload); boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException; diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index fee17bc..0ec8639 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -169,6 +169,16 @@ public interface ExtendedClient extends Flushable, Closeable { */ ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException; + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails + */ + ExtendedClient newIndex(String index, Settings settings) throws IOException; + /** * Create a new index. * @@ -364,9 +374,11 @@ public interface ExtendedClient extends Flushable, Closeable { * @param index the index * @param key the key of the value to be updated * @param value the new value + * @param timeout timeout + * @param timeUnit time unit * @throws IOException if update index setting failed */ - void updateIndexSetting(String index, String key, Object value) throws IOException; + void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; /** * Resolve alias. @@ -386,11 +398,11 @@ public interface ExtendedClient extends Flushable, Closeable { String resolveMostRecentIndex(String alias); /** - * Get all index filters. + * Get all aliases. * @param index the index - * @return map of index filters + * @return map of index aliases */ - Map getIndexFilters(String index); + Map getAliases(String index); /** * Shift from one index to another. diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java index d92bca3..4e2dca9 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexAliasAdder.java @@ -1,9 +1,9 @@ package org.xbib.elx.api; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; @FunctionalInterface public interface IndexAliasAdder { - void addIndexAlias(IndicesAliasesRequestBuilder builder, String index, String alias); + void addIndexAlias(IndicesAliasesRequest request, String index, String alias); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/package-info.java b/elx-api/src/main/java/org/xbib/elx/api/package-info.java index 7991a43..03fd0e3 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/package-info.java +++ b/elx-api/src/main/java/org/xbib/elx/api/package-info.java @@ -1,4 +1,4 @@ /** - * The API of the Elasticsearch extensions. + * The API of the extended Elasticsearch clients. */ package org.xbib.elx.api; diff --git a/elx-common/build.gradle b/elx-common/build.gradle index e794a8b..25b9d83 100644 --- a/elx-common/build.gradle +++ b/elx-common/build.gradle @@ -1,8 +1,6 @@ dependencies { compile project(':elx-api') compile "org.xbib:guice:${project.property('xbib-guice.version')}" - // add all dependencies to runtime source set, even that which are excluded by Elasticsearch jar, - // for metaprogramming. We are in Groovyland. runtime "com.vividsolutions:jts:${project.property('jts.version')}" runtime "com.github.spullara.mustache.java:compiler:${project.property('mustache.version')}" runtime "net.java.dev.jna:jna:${project.property('jna.version')}" diff --git a/elx-common/build.gradle~ b/elx-common/build.gradle~ deleted file mode 100644 index 99099fb..0000000 --- a/elx-common/build.gradle~ +++ /dev/null @@ -1,65 +0,0 @@ -buildscript { - repositories { - jcenter() - maven { - url 'http://xbib.org/repository' - } - } - dependencies { - classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" - } -} - -apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' - -configurations { - main - tests -} - -dependencies { - compile project(':api') - compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" - compileOnly "org.apache.logging.log4j:log4j-api:${project.property('log4j.version')}" - testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" - testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" -} - -jar { - baseName "${rootProject.name}-common" -} - -/* -task testJar(type: Jar, dependsOn: testClasses) { - baseName = "${project.archivesBaseName}-tests" - from sourceSets.test.output -} -*/ - -artifacts { - main jar - tests testJar - archives sourcesJar, javadocJar -} - -test { - enabled = false - jvmArgs "-javaagent:" + configurations.alpnagent.asPath - systemProperty 'path.home', project.buildDir.absolutePath - testLogging { - showStandardStreams = true - exceptionFormat = 'full' - } -} - -randomizedTest { - enabled = false -} - -esTest { - // test with the jars, not the classes, for security manager - // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files - systemProperty 'tests.security.manager', 'true' -} -esTest.dependsOn jar, testJar - diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 21f2730..5ea81ae 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -9,29 +9,29 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; -import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; import org.elasticsearch.action.admin.indices.flush.FlushAction; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; -import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequestBuilder; +import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; @@ -46,13 +46,16 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -62,6 +65,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -82,14 +86,15 @@ import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedMap; import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.ExecutionException; @@ -159,6 +164,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; + protected abstract void closeClient() throws IOException; + protected AbstractExtendedClient() { closed = new AtomicBoolean(false); } @@ -166,9 +173,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public AbstractExtendedClient setClient(ElasticsearchClient client) { this.client = client; - this.bulkMetric = new DefaultBulkMetric(); - bulkMetric.start(); - this.bulkController = new DefaultBulkController(this, bulkMetric); return this; } @@ -192,10 +196,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (client == null) { client = createClient(settings); } - if (bulkMetric != null) { - bulkMetric.start(); + if (bulkMetric == null) { + this.bulkMetric = new DefaultBulkMetric(); + this.bulkMetric.init(settings); } - if (bulkController != null) { + if (bulkController == null) { + this.bulkController = new DefaultBulkController(this, bulkMetric); bulkController.init(settings); } return this; @@ -213,14 +219,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { ensureActive(); if (closed.compareAndSet(false, true)) { if (bulkMetric != null) { - logger.info("closing bulk metric before bulk controller (for precise measurement)"); + logger.info("closing bulk metric"); bulkMetric.close(); } if (bulkController != null) { logger.info("closing bulk controller"); bulkController.close(); } - logger.info("shutdown complete"); + closeClient(); } } @@ -228,9 +234,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public String getClusterName() { ensureActive(); try { - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(client, ClusterStateAction.INSTANCE).all(); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); return clusterStateResponse.getClusterName().value(); } catch (ElasticsearchTimeoutException e) { logger.warn(e.getMessage(), e); @@ -280,7 +286,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient newIndex(String index) { + public ExtendedClient newIndex(String index) throws IOException { return newIndex(index, Settings.EMPTY, (Map) null); } @@ -288,31 +294,35 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException { return newIndex(index, Settings.settingsBuilder().loadFromStream(".json", settings).build(), - JsonXContent.jsonXContent.createParser(mapping).mapOrdered()); + mapping != null ? JsonXContent.jsonXContent.createParser(mapping).mapOrdered() : null); + } + + @Override + public ExtendedClient newIndex(String index, Settings settings) throws IOException { + return newIndex(index, settings, (Map) null); } @Override public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException { return newIndex(index, settings, - JsonXContent.jsonXContent.createParser(mapping).mapOrdered()); + mapping != null ? JsonXContent.jsonXContent.createParser(mapping).mapOrdered() : null); } @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) { + public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { ensureActive(); if (index == null) { logger.warn("no index name given to create index"); return this; } - CreateIndexRequestBuilder createIndexRequestBuilder = - new CreateIndexRequestBuilder(client, CreateIndexAction.INSTANCE).setIndex(index); + CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); if (settings != null) { - createIndexRequestBuilder.setSettings(settings); + createIndexRequest.settings(settings); } if (mapping != null) { - createIndexRequestBuilder.addMapping(TYPE_NAME, mapping); + createIndexRequest.mapping(TYPE_NAME, mapping); } - CreateIndexResponse createIndexResponse = createIndexRequestBuilder.execute().actionGet(); + CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); logger.info("index {} created: {}", index, createIndexResponse); return this; } @@ -329,9 +339,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { logger.warn("no index name given to delete index"); return this; } - DeleteIndexRequestBuilder deleteIndexRequestBuilder = - new DeleteIndexRequestBuilder(client, DeleteIndexAction.INSTANCE, index); - deleteIndexRequestBuilder.execute().actionGet(); + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index); + client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); return this; } @@ -371,12 +380,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create).source(source)); + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source)); } @Override public ExtendedClient index(String index, String id, boolean create, String source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create).source(source.getBytes(StandardCharsets.UTF_8))); + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source.getBytes(StandardCharsets.UTF_8))); } @Override @@ -425,12 +436,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { ensureActive(); ensureIndexGiven(index); - RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, new RecoveryRequest(index)).actionGet(); + RecoveryRequest recoveryRequest = new RecoveryRequest(); + recoveryRequest.indices(index); + recoveryRequest.activeOnly(true); + RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, recoveryRequest).actionGet(); int shards = response.getTotalShards(); TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() + .indices(new String[]{index}) + .waitForActiveShards(shards) + .timeout(timeout); ClusterHealthResponse healthResponse = - client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest(index) - .waitForActiveShards(shards).timeout(timeout)).actionGet(); + client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { logger.error("timeout waiting for recovery"); return false; @@ -485,7 +502,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations if (level > 0) { - updateIndexSetting(index, "number_of_replicas", level); + updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); waitForRecovery(index, maxWaitTime, timeUnit); } return this; @@ -528,25 +545,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return this; } - @Override - public String resolveAlias(String alias) { - ensureActive(); - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); - if (!getAliasesResponse.getAliases().isEmpty()) { - return getAliasesResponse.getAliases().keys().iterator().next().value; - } - return alias; - } - @Override public String resolveMostRecentIndex(String alias) { ensureActive(); if (alias == null) { return null; } - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias); + GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); Set indices = new TreeSet<>(Collections.reverseOrder()); for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { @@ -558,10 +564,24 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return indices.isEmpty() ? alias : indices.iterator().next(); } + public Map getAliases(String index) { + if (index == null) { + return Collections.emptyMap(); + } + GetAliasesRequest getAliasesRequest = new GetAliasesRequest().indices(index); + return getFilters(client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet()); + } + @Override - public Map getIndexFilters(String index) { - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, GetAliasesAction.INSTANCE); - return getFilters(getAliasesRequestBuilder.setIndices(index).execute().actionGet()); + public String resolveAlias(String alias) { + ensureActive(); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); + clusterStateRequest.metaData(true); + ClusterStateResponse clusterStateResponse = + client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + SortedMap map = clusterStateResponse.getState().getMetaData().getAliasAndIndexLookup(); + AliasOrIndex aliasOrIndex = map.get(alias); + return aliasOrIndex != null ? aliasOrIndex.getIndices().iterator().next().getIndex() : null; } @Override @@ -593,60 +613,81 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, IndexAliasAdder adder) { ensureActive(); + if (index == null) { + return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to + } if (index.equals(fullIndexName)) { return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to } - // two situations: 1. there is a new alias 2. there is already an old index with the alias + waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); + // two situations: 1. a new alias 2. there is already an old index with the alias String oldIndex = resolveAlias(index); - final Map oldFilterMap = oldIndex.equals(index) ? null : getIndexFilters(oldIndex); - final List newAliases = new LinkedList<>(); - final List moveAliases = new LinkedList<>(); - IndicesAliasesRequestBuilder requestBuilder = new IndicesAliasesRequestBuilder(client, IndicesAliasesAction.INSTANCE); - if (oldFilterMap == null || !oldFilterMap.containsKey(index)) { - // never apply a filter for trunk index name - requestBuilder.addAlias(fullIndexName, index); + Map oldAliasMap = index.equals(oldIndex) ? null : getAliases(oldIndex); + logger.debug("old index = {} old alias map = {}", oldIndex, oldAliasMap); + final List newAliases = new ArrayList<>(); + final List moveAliases = new ArrayList<>(); + IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); + if (oldAliasMap == null || !oldAliasMap.containsKey(index)) { + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + fullIndexName, index)); newAliases.add(index); } // move existing aliases - if (oldFilterMap != null) { - for (Map.Entry entry : oldFilterMap.entrySet()) { + if (oldAliasMap != null) { + for (Map.Entry entry : oldAliasMap.entrySet()) { String alias = entry.getKey(); String filter = entry.getValue(); - requestBuilder.removeAlias(oldIndex, alias); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.REMOVE, + oldIndex, alias)); if (filter != null) { - requestBuilder.addAlias(fullIndexName, alias, filter); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + fullIndexName, index).filter(filter)); } else { - requestBuilder.addAlias(fullIndexName, alias); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + fullIndexName, index)); } moveAliases.add(alias); } } // a list of aliases that should be added, check if new or old if (additionalAliases != null) { - for (String extraAlias : additionalAliases) { - if (oldFilterMap == null || !oldFilterMap.containsKey(extraAlias)) { + for (String additionalAlias : additionalAliases) { + if (oldAliasMap == null || !oldAliasMap.containsKey(additionalAlias)) { // index alias adder only active on extra aliases, and if alias is new if (adder != null) { - adder.addIndexAlias(requestBuilder, fullIndexName, extraAlias); + adder.addIndexAlias(indicesAliasesRequest, fullIndexName, additionalAlias); } else { - requestBuilder.addAlias(fullIndexName, extraAlias); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + fullIndexName, additionalAlias)); } - newAliases.add(extraAlias); + newAliases.add(additionalAlias); } else { - String filter = oldFilterMap.get(extraAlias); - requestBuilder.removeAlias(oldIndex, extraAlias); + String filter = oldAliasMap.get(additionalAlias); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.REMOVE, + oldIndex, additionalAlias)); if (filter != null) { - requestBuilder.addAlias(fullIndexName, extraAlias, filter); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + fullIndexName, additionalAlias).filter(filter)); } else { - requestBuilder.addAlias(fullIndexName, extraAlias); + indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + fullIndexName, additionalAlias)); } - moveAliases.add(extraAlias); + moveAliases.add(additionalAlias); } } } - if (!newAliases.isEmpty() || !moveAliases.isEmpty()) { - logger.info("new aliases = {}, moved aliases = {}", newAliases, moveAliases); - requestBuilder.execute().actionGet(); + if (!indicesAliasesRequest.getAliasActions().isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (IndicesAliasesRequest.AliasActions aliasActions : indicesAliasesRequest.getAliasActions()) { + sb.append("[").append(aliasActions.actionType().name()) + .append(",indices=").append(Arrays.asList(aliasActions.indices())) + .append(",aliases=").append(Arrays.asList(aliasActions.aliases())).append("]"); + } + logger.debug("indices alias request = {}", sb.toString()); + IndicesAliasesResponse indicesAliasesResponse = + client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); + logger.debug("response isAcknowledged = {}", + indicesAliasesResponse.isAcknowledged()); } return new SuccessIndexShiftResult(moveAliases, newAliases); } @@ -711,13 +752,15 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public Long mostRecentDocument(String index, String timestampfieldname) { ensureActive(); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); - SearchResponse searchResponse = searchRequestBuilder.setIndices(index) - .addField(timestampfieldname) - .setSize(1) - .addSort(sort) - .execute().actionGet(); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.field(timestampfieldname); + sourceBuilder.size(1); + sourceBuilder.sort(sort); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.source(sourceBuilder); + SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); if (searchResponse.getHits().getHits().length == 1) { SearchHit hit = searchResponse.getHits().getHits()[0]; if (hit.getFields().get(timestampfieldname) != null) { @@ -741,11 +784,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ForceMergeRequestBuilder forceMergeRequestBuilder = - new ForceMergeRequestBuilder(client, ForceMergeAction.INSTANCE); - forceMergeRequestBuilder.setIndices(index); + ForceMergeRequest forceMergeRequest = new ForceMergeRequest(); + forceMergeRequest.indices(index); try { - forceMergeRequestBuilder.execute().get(timeout.getMillis(), TimeUnit.MILLISECONDS); + client.execute(ForceMergeAction.INSTANCE, forceMergeRequest).get(timeout.getMillis(), TimeUnit.MILLISECONDS); return true; } catch (TimeoutException e) { logger.error("timeout"); @@ -794,7 +836,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public void updateIndexSetting(String index, String key, Object value) throws IOException { + public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { ensureActive(); if (index == null) { throw new IOException("no index name given"); @@ -808,7 +850,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { Settings.Builder updateSettingsBuilder = Settings.settingsBuilder(); updateSettingsBuilder.put(key, value.toString()); UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) - .settings(updateSettingsBuilder); + .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit)); client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); } @@ -845,9 +887,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public void checkMapping(String index) { ensureActive(); - GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) - .setIndices(index); - GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet(); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index); + GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); ImmutableOpenMap> map = getMappingsResponse.getMappings(); map.keys().forEach((Consumer>) stringObjectCursor -> { ImmutableOpenMap mappings = map.get(stringObjectCursor.value); @@ -930,7 +971,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .setQuery(queryBuilder) .execute() .actionGet(); - fields.put(path, searchResponse.getHits().totalHits()); + fields.put(path, searchResponse.getHits().getTotalHits()); } } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index ca705c4..30d5b52 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -107,7 +107,8 @@ public class DefaultBulkController implements BulkController { startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds); stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds); if (startRefreshIntervalInSeconds != 0L) { - client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s"); + client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s", + 30L, TimeUnit.SECONDS); } } } @@ -193,7 +194,8 @@ public class DefaultBulkController implements BulkController { if (indexNames.contains(index)) { Long secs = stopBulkRefreshIntervals.get(index); if (secs != null && secs != 0L) { - client.updateIndexSetting(index, "refresh_interval", secs + "s"); + client.updateIndexSetting(index, "refresh_interval", secs + "s", + 30L, TimeUnit.SECONDS); } indexNames.remove(index); } @@ -214,7 +216,8 @@ public class DefaultBulkController implements BulkController { for (String index : indexNames) { Long secs = stopBulkRefreshIntervals.get(index); if (secs != null && secs != 0L) - client.updateIndexSetting(index, "refresh_interval", secs + "s"); + client.updateIndexSetting(index, "refresh_interval", secs + "s", + 30L, TimeUnit.SECONDS); } indexNames.clear(); } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 28dbb45..224f507 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -49,7 +49,7 @@ public class DefaultBulkProcessor implements BulkProcessor { this.executionIdGen = new AtomicLong(); this.closed = false; this.bulkActions = bulkActions; - this.bulkSize = bulkSize.bytes(); + this.bulkSize = bulkSize.getBytes(); this.bulkRequest = new BulkRequest(); this.bulkRequestHandler = concurrentRequests == 0 ? new SyncBulkRequestHandler(client, listener) : @@ -133,7 +133,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @return his bulk processor */ @Override - public DefaultBulkProcessor add(ActionRequest request) { + public DefaultBulkProcessor add(ActionRequest request) { return add(request, null); } @@ -145,7 +145,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @return his bulk processor */ @Override - public DefaultBulkProcessor add(ActionRequest request, Object payload) { + public DefaultBulkProcessor add(ActionRequest request, Object payload) { internalAdd(request, payload); return this; } @@ -176,7 +176,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private void ensureOpen() { if (closed) { - throw new IllegalStateException("bulk process already closed"); + throw new IllegalStateException("bulk processor already closed"); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java index 58e303d..5cc6cdd 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -6,10 +6,11 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import java.io.IOException; import java.util.concurrent.TimeUnit; /** - * Mock client, it does not perform any actions on a cluster. Useful for testing. + * A mocked client, it does not perform any actions on a cluster. Useful for testing. */ public class MockExtendedClient extends AbstractExtendedClient { @@ -28,6 +29,10 @@ public class MockExtendedClient extends AbstractExtendedClient { return null; } + @Override + protected void closeClient() { + } + @Override public MockExtendedClient index(String index, String id, boolean create, String source) { return this; diff --git a/elx-common/src/test/java/org/elasticsearch/node/MockNode.java b/elx-common/src/test/java/org/elasticsearch/node/MockNode.java deleted file mode 100644 index aad8b8b..0000000 --- a/elx-common/src/test/java/org/elasticsearch/node/MockNode.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.elasticsearch.node; - -import org.elasticsearch.Version; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.elasticsearch.plugins.Plugin; - -import java.util.ArrayList; -import java.util.Collection; - -public class MockNode extends Node { - - public MockNode() { - super(Settings.EMPTY); - } - - public MockNode(Settings settings) { - super(settings); - } - - public MockNode(Settings settings, Collection> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); - } - - public MockNode(Settings settings, Class classpathPlugin) { - this(settings, list(classpathPlugin)); - } - - private static Collection> list(Class classpathPlugin) { - Collection> list = new ArrayList<>(); - list.add(classpathPlugin); - return list; - } -} diff --git a/elx-common/src/test/java/org/elasticsearch/node/package-info.java b/elx-common/src/test/java/org/elasticsearch/node/package-info.java deleted file mode 100644 index 8ffed8c..0000000 --- a/elx-common/src/test/java/org/elasticsearch/node/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.elasticsearch.node; \ No newline at end of file diff --git a/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java deleted file mode 100644 index 75cdc29..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/SimpleTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.xbib.elx.common; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.query.QueryBuilders.matchQuery; -import static org.junit.Assert.assertEquals; - -import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder; -import org.elasticsearch.action.index.IndexAction; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexNotFoundException; -import org.junit.Test; - -public class SimpleTest extends NodeTestUtils { - - protected Settings getNodeSettings() { - return settingsBuilder() - .put(super.getNodeSettings()) - .put("index.analysis.analyzer.default.filter.0", "lowercase") - .put("index.analysis.analyzer.default.filter.1", "trim") - .put("index.analysis.analyzer.default.tokenizer", "keyword") - .build(); - } - - @Test - public void test() throws Exception { - try { - DeleteIndexRequestBuilder deleteIndexRequestBuilder = - new DeleteIndexRequestBuilder(client("1"), DeleteIndexAction.INSTANCE, "test"); - deleteIndexRequestBuilder.execute().actionGet(); - } catch (IndexNotFoundException e) { - // ignore if index not found - } - IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(client("1"), IndexAction.INSTANCE); - indexRequestBuilder - .setIndex("test") - .setType("test") - .setId("1") - .setSource(jsonBuilder().startObject().field("field", - "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()) - .setRefresh(true) - .execute() - .actionGet(); - String doc = client("1").prepareSearch("test") - .setTypes("test") - .setQuery(matchQuery("field", - "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8")) - .execute() - .actionGet() - .getHits().getAt(0).getSourceAsString(); - - assertEquals(doc, - "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}"); - } -} diff --git a/elx-common/src/test/java/org/xbib/elx/common/package-info.java b/elx-common/src/test/java/org/xbib/elx/common/package-info.java deleted file mode 100644 index 9a9e4ce..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.xbib.elx.common; \ No newline at end of file diff --git a/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java similarity index 90% rename from elx-common/src/test/java/org/xbib/elx/common/AliasTest.java rename to elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java index 419da0e..b1b7f95 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.common; +package org.xbib.elx.common.test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -9,7 +9,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequestBuilder; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.client.Client; @@ -27,7 +26,7 @@ import java.util.regex.Pattern; /** * */ -public class AliasTest extends NodeTestUtils { +public class AliasTest extends TestBase { private static final Logger logger = LogManager.getLogger(AliasTest.class.getName()); @@ -71,9 +70,9 @@ public class AliasTest extends NodeTestUtils { indicesAliasesRequest.addAliasAction(aliasAction); client.admin().indices().aliases(indicesAliasesRequest).actionGet(); - GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client, - GetAliasesAction.INSTANCE); - GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet(); + GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); + getAliasesRequest.aliases(alias); + GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); Set result = new TreeSet<>(Collections.reverseOrder()); for (ObjectCursor indexName : getAliasesResponse.getAliases().keys()) { diff --git a/elx-node/src/test/java/org/xbib/elx/node/ClusterBlockTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java similarity index 90% rename from elx-node/src/test/java/org/xbib/elx/node/ClusterBlockTest.java rename to elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java index 23cbbe3..dae869f 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ClusterBlockTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.node; +package org.xbib.elx.common.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -10,17 +10,16 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; -public class ClusterBlockTest extends NodeTestUtils { +public class ClusterBlockTest extends TestBase { private static final Logger logger = LogManager.getLogger("test"); @Before public void startNodes() { try { - setClusterName(); + setClusterName("test-cluster" + System.getProperty("user.name")); startNode("1"); // do not wait for green health state logger.info("ready"); diff --git a/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java similarity index 67% rename from elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java rename to elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java index 8474c1c..cbe7972 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/MockExtendedClientProviderTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java @@ -1,6 +1,9 @@ -package org.xbib.elx.common; +package org.xbib.elx.common.test; import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.MockExtendedClient; +import org.xbib.elx.common.MockExtendedClientProvider; import java.io.IOException; diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java new file mode 100644 index 0000000..e60c981 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockNode.java @@ -0,0 +1,12 @@ +package org.xbib.elx.common.test; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; + +public class MockNode extends Node { + + public MockNode(Settings settings) { + super(settings); + } + +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java similarity index 93% rename from elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java rename to elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java index 248b906..7933343 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/NetworkTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.common; +package org.xbib.elx.common.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,7 +15,7 @@ public class NetworkTest { @Test public void testNetwork() throws Exception { - // walk very slowly over all interfaces + // walk over all found interfaces (this is slow - multicast/pings are performed) Enumeration nets = NetworkInterface.getNetworkInterfaces(); for (NetworkInterface netint : Collections.list(nets)) { System.out.println("checking network interface = " + netint.getName()); diff --git a/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java similarity index 93% rename from elx-common/src/test/java/org/xbib/elx/common/SearchTest.java rename to elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index 63892d0..6e23f0b 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.common; +package org.xbib.elx.common.test; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkAction; @@ -15,7 +15,7 @@ import org.junit.Test; import static org.junit.Assert.assertTrue; -public class SearchTest extends NodeTestUtils { +public class SearchTest extends TestBase { @Test public void testSearch() throws Exception { @@ -35,7 +35,8 @@ public class SearchTest extends NodeTestUtils { .field("user8", "joerg") .field("user9", "joerg") .field("rowcount", i) - .field("rs", 1234)); + .field("rs", 1234) + .endObject()); builder.add(indexRequest); } client.bulk(builder.request()).actionGet(); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java new file mode 100644 index 0000000..5948daa --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java @@ -0,0 +1,61 @@ +package org.xbib.elx.common.test; + +import static org.junit.Assert.assertEquals; + +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.junit.Test; + +public class SimpleTest extends TestBase { + + @Test + public void test() throws Exception { + try { + DeleteIndexRequest deleteIndexRequest = + new DeleteIndexRequest().indices("test"); + client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); + } catch (IndexNotFoundException e) { + // ignore if index not found + } + Settings indexSettings = Settings.settingsBuilder() + .put(super.getNodeSettings()) + .put("index.analysis.analyzer.default.filter.0", "lowercase") + .put("index.analysis.analyzer.default.filter.1", "trim") + .put("index.analysis.analyzer.default.tokenizer", "keyword") + .build(); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(); + createIndexRequest.index("test").settings(indexSettings); + client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index("test").type("test").id("1") + .source(XContentFactory.jsonBuilder().startObject().field("field", + "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()); + client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); + RefreshRequest refreshRequest = new RefreshRequest(); + refreshRequest.indices("test"); + client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(QueryBuilders.matchQuery("field", + "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8")); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("test").types("test"); + searchRequest.source(builder); + String doc = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet() + .getHits().getAt(0).getSourceAsString(); + assertEquals(doc, + "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}"); + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java similarity index 65% rename from elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java rename to elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java index 7faed8d..12dc194 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/NodeTestUtils.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java @@ -1,6 +1,4 @@ -package org.xbib.elx.node; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +package org.xbib.elx.common.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -10,16 +8,17 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.junit.After; import org.junit.Before; -import org.xbib.elx.common.util.NetworkUtils; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -31,47 +30,34 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; -public class NodeTestUtils { +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class TestBase { private static final Logger logger = LogManager.getLogger("test"); - private static Random random = new Random(); + private static final Random random = new Random(); - private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); private Map nodes = new HashMap<>(); private Map clients = new HashMap<>(); - private AtomicInteger counter = new AtomicInteger(); + private String cluster; - protected String clusterName; + private String host; - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } + private int port; @Before public void startNodes() { try { logger.info("starting"); - setClusterName(); + setClusterName("test-cluster-" + System.getProperty("user.name")); startNode("1"); + findNodeAddress(); try { ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) @@ -83,6 +69,12 @@ public class NodeTestUtils { } catch (ElasticsearchTimeoutException e) { throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } catch (Throwable t) { logger.error("startNodes failed", t); } @@ -107,24 +99,34 @@ public class NodeTestUtils { } } - protected void setClusterName() { - this.clusterName = "test-helper-cluster-" - + NetworkUtils.getLocalAddress().getHostName() - + "-" + System.getProperty("user.name") - + "-" + counter.incrementAndGet(); + protected void setClusterName(String cluster) { + this.cluster = cluster; + } + + protected String getClusterName() { + return cluster; + } + + protected Settings getTransportSettings() { + return settingsBuilder() + .put("host", host) + .put("port", port) + .put("cluster.name", cluster) + .put("path.home", getHome()) + .build(); } protected Settings getNodeSettings() { return settingsBuilder() - .put("cluster.name", clusterName) - .put("cluster.routing.schedule", "50ms") - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.zen.multicast.enabled", true) - .put("discovery.zen.multicast.ping_timeout", "5s") - .put("http.enabled", true) - .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) - .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low - .put("index.number_of_replicas", 0) + .put("cluster.name", cluster) + //.put("cluster.routing.schedule", "50ms") + //.put("cluster.routing.allocation.disk.threshold_enabled", false) + //.put("discovery.zen.multicast.enabled", true) + //.put("discovery.zen.multicast.ping_timeout", "5s") + //.put("http.enabled", true) + //.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + //.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + //.put("index.number_of_replicas", 0) .put("path.home", getHome()) .build(); } @@ -133,14 +135,48 @@ public class NodeTestUtils { return System.getProperty("path.home", System.getProperty("user.dir")); } - public void startNode(String id) { + protected void startNode(String id) { buildNode(id).start(); } - public AbstractClient client(String id) { + protected AbstractClient client(String id) { return clients.get(id); } + protected void findNodeAddress() { + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + protected String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + private void closeNodes() { logger.info("closing all clients"); for (AbstractClient client : clients.values()) { @@ -157,26 +193,20 @@ public class NodeTestUtils { logger.info("all nodes closed"); } - private Node buildNode(String id) { - Settings nodeSettings = settingsBuilder() - .put(getNodeSettings()) - .put("name", id) - .build(); - logger.info("settings={}", nodeSettings.getAsMap()); - Node node = new MockNode(nodeSettings); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - logger.info("clients={}", clients); - return node; - } + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java similarity index 69% rename from elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java rename to elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java index 783b440..1bb681c 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/WildcardTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java @@ -1,18 +1,23 @@ -package org.xbib.elx.common; +package org.xbib.elx.common.test; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.Client; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Test; import java.io.IOException; -public class WildcardTest extends NodeTestUtils { +public class WildcardTest extends TestBase { - protected Settings getNodeSettings() { + /*protected Settings getNodeSettings() { return Settings.settingsBuilder() .put(super.getNodeSettings()) .put("cluster.routing.allocation.disk.threshold_enabled", false) @@ -21,7 +26,7 @@ public class WildcardTest extends NodeTestUtils { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - } + }*/ @Test public void testWildcard() throws Exception { @@ -42,15 +47,19 @@ public class WildcardTest extends NodeTestUtils { } private void index(Client client, String id, String fieldValue) throws IOException { - client.index(new IndexRequest("index", "type", id) - .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject()) - .refresh(true)).actionGet(); + client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id) + .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())).actionGet(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); } private long count(Client client, QueryBuilder queryBuilder) { - return client.prepareSearch("index").setTypes("type") - .setQuery(queryBuilder) - .execute().actionGet().getHits().getTotalHits(); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(queryBuilder); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("index"); + searchRequest.types("type"); + searchRequest.source(builder); + return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); } private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java b/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java new file mode 100644 index 0000000..5a27aff --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.common.test; \ No newline at end of file diff --git a/elx-http/build.gradle~ b/elx-http/build.gradle~ deleted file mode 100644 index da70162..0000000 --- a/elx-http/build.gradle~ +++ /dev/null @@ -1,65 +0,0 @@ -buildscript { - repositories { - jcenter() - maven { - url 'http://xbib.org/repository' - } - } - dependencies { - classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" - } -} - -apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' - -configurations { - main - tests -} - -dependencies { - compile project(':common') - compile "org.xbib:netty-http-client:${project.property('xbib-netty-http-client.version')}" - testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" - testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" -} - -jar { - baseName "${rootProject.name}-common" -} - -/* -task testJar(type: Jar, dependsOn: testClasses) { - baseName = "${project.archivesBaseName}-tests" - from sourceSets.test.output -} -*/ - -artifacts { - main jar - tests testJar - archives sourcesJar, javadocJar -} - -test { - enabled = true - include '**/SimpleTest.*' - testLogging { - showStandardStreams = true - exceptionFormat = 'full' - } -} - -randomizedTest { - enabled = false -} - -esTest { - enabled = true - // test with the jars, not the classes, for security manager - // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files - systemProperty 'tests.security.manager', 'true' - // maybe we like some extra security policy for our code - systemProperty 'tests.security.policy', '/extra-security.policy' -} -esTest.dependsOn jar, testJar diff --git a/elx-node/build.gradle~ b/elx-node/build.gradle~ deleted file mode 100644 index 0da2929..0000000 --- a/elx-node/build.gradle~ +++ /dev/null @@ -1,65 +0,0 @@ -buildscript { - repositories { - jcenter() - maven { - url 'http://xbib.org/repository' - } - } - dependencies { - classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.3.4" - } -} - -apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' - -configurations { - main - tests -} - -dependencies { - compile project(':common') - testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" - testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" -} - -jar { - baseName "${rootProject.name}-node" -} - -/* -task testJar(type: Jar, dependsOn: testClasses) { - baseName = "${project.archivesBaseName}-tests" - from sourceSets.test.output -} -*/ - -artifacts { - main jar - tests testJar - archives sourcesJar, javadocJar -} - -test { - enabled = false - jvmArgs "-javaagent:" + configurations.alpnagent.asPath - systemProperty 'path.home', projectDir.absolutePath - testLogging { - showStandardStreams = true - exceptionFormat = 'full' - } -} - -randomizedTest { - enabled = false -} - - -esTest { - // test with the jars, not the classes, for security manager - // classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files - systemProperty 'tests.security.manager', 'true' - // maybe we like some extra security policy for our code - systemProperty 'tests.security.policy', '/extra-security.policy' -} -esTest.dependsOn jar, testJar diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java index 7eca86e..217fbef 100644 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java +++ b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java @@ -34,7 +34,7 @@ public class ExtendedNodeClient extends AbstractExtendedClient { .put("node.data", false) .build(); logger.info("creating node client on {} with effective settings {}", - version, effectiveSettings.toString()); + version, effectiveSettings.getAsMap()); Collection> plugins = Collections.emptyList(); this.node = new BulkNode(new Environment(effectiveSettings), plugins); try { @@ -48,15 +48,10 @@ public class ExtendedNodeClient extends AbstractExtendedClient { } @Override - public void close() throws IOException { - super.close(); - try { - if (node != null) { - logger.debug("closing node..."); - node.close(); - } - } catch (Exception e) { - logger.error(e.getMessage(), e); + protected void closeClient() { + if (node != null) { + logger.debug("closing node..."); + node.close(); } } diff --git a/elx-node/src/test/java/org/elasticsearch/node/MockNode.java b/elx-node/src/test/java/org/elasticsearch/node/MockNode.java deleted file mode 100644 index 1de4c2f..0000000 --- a/elx-node/src/test/java/org/elasticsearch/node/MockNode.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.elasticsearch.node; - -import org.elasticsearch.Version; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.elasticsearch.plugins.Plugin; - -import java.util.ArrayList; -import java.util.Collection; - -public class MockNode extends Node { - - public MockNode(Settings settings) { - super(settings); - } - - public MockNode(Settings settings, Collection> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); - } - - public MockNode(Settings settings, Class classpathPlugin) { - this(settings, list(classpathPlugin)); - } - - private static Collection> list(Class classpathPlugin) { - Collection> list = new ArrayList<>(); - list.add(classpathPlugin); - return list; - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java deleted file mode 100644 index 9fb687a..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/IndexShiftTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.xbib.elx.node; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -@Ignore -public class IndexShiftTest extends NodeTestUtils { - - private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); - - @Test - public void testIndexShift() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - try { - client.newIndex("test1234"); - for (int i = 0; i < 1; i++) { - client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.refreshIndex("test1234"); - - List simpleAliases = Arrays.asList("a", "b", "c"); - client.shiftIndex("test", "test1234", simpleAliases); - - client.newIndex("test5678"); - for (int i = 0; i < 1; i++) { - client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.refreshIndex("test5678"); - - simpleAliases = Arrays.asList("d", "e", "f"); - client.shiftIndex("test", "test5678", simpleAliases, (builder, index, alias) -> - builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); - Map indexFilters = client.getIndexFilters("test5678"); - logger.info("aliases of index test5678 = {}", indexFilters); - assertTrue(indexFilters.containsKey("a")); - assertTrue(indexFilters.containsKey("b")); - assertTrue(indexFilters.containsKey("c")); - assertTrue(indexFilters.containsKey("d")); - assertTrue(indexFilters.containsKey("e")); - - Map aliases = client.getIndexFilters(client.resolveAlias("test")); - logger.info("aliases of alias test = {}", aliases); - assertTrue(aliases.containsKey("a")); - assertTrue(aliases.containsKey("b")); - assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("d")); - assertTrue(aliases.containsKey("e")); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.waitForResponses(30L, TimeUnit.SECONDS); - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java similarity index 97% rename from elx-node/src/test/java/org/xbib/elx/node/ClientTest.java rename to elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java index 5aaae33..dc147b0 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.node; +package org.xbib.elx.node.test; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.junit.Assert.assertEquals; @@ -22,14 +22,16 @@ import org.junit.Before; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import org.xbib.elx.node.ExtendedNodeClient; +import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public class ClientTest extends NodeTestUtils { +public class ClientTest extends TestBase { - private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); private static final Long ACTIONS = 25000L; @@ -144,7 +146,7 @@ public class ClientTest extends NodeTestUtils { logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); final ExtendedNodeClient client = ClientBuilder.builder(client("1")) .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); diff --git a/elx-node/src/test/java/org/xbib/elx/node/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java similarity index 69% rename from elx-node/src/test/java/org/xbib/elx/node/DuplicateIDTest.java rename to elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index d2126e5..9ea5c40 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -1,23 +1,25 @@ -package org.xbib.elx.node; +package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.junit.Ignore; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import org.xbib.elx.node.ExtendedNodeClient; +import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.*; -public class DuplicateIDTest extends NodeTestUtils { +public class DuplicateIDTest extends TestBase { - private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; @@ -38,11 +40,13 @@ public class DuplicateIDTest extends NodeTestUtils { client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(QueryBuilders.matchAllQuery()); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("test"); + searchRequest.types("test"); + searchRequest.source(builder); + long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java new file mode 100644 index 0000000..8b31d89 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -0,0 +1,111 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.metadata.AliasAction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.query.QueryBuilders; +import org.junit.Test; +import org.xbib.elx.api.IndexShiftResult; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.node.ExtendedNodeClient; +import org.xbib.elx.node.ExtendedNodeClientProvider; + +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class IndexShiftTest extends TestBase { + + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + + @Test + public void testIndexShift() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + client.newIndex("test1234", settings); + for (int i = 0; i < 1; i++) { + client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + + IndexShiftResult indexShiftResult = + client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); + + assertTrue(indexShiftResult.getNewAliases().contains("a")); + assertTrue(indexShiftResult.getNewAliases().contains("b")); + assertTrue(indexShiftResult.getNewAliases().contains("c")); + assertTrue(indexShiftResult.getMovedAliases().isEmpty()); + + Map aliases = client.getAliases("test1234"); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("test")); + + String resolved = client.resolveAlias("test"); + aliases = client.getAliases(resolved); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("test")); + + client.newIndex("test5678", settings); + for (int i = 0; i < 1; i++) { + client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + } + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + + indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + index, alias).filter(QueryBuilders.termQuery("my_key", alias))) + ); + assertTrue(indexShiftResult.getNewAliases().contains("d")); + assertTrue(indexShiftResult.getNewAliases().contains("e")); + assertTrue(indexShiftResult.getNewAliases().contains("f")); + assertTrue(indexShiftResult.getMovedAliases().contains("a")); + assertTrue(indexShiftResult.getMovedAliases().contains("b")); + assertTrue(indexShiftResult.getMovedAliases().contains("c")); + + aliases = client.getAliases("test5678"); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("d")); + assertTrue(aliases.containsKey("e")); + assertTrue(aliases.containsKey("f")); + + resolved = client.resolveAlias("test"); + aliases = client.getAliases(resolved); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("d")); + assertTrue(aliases.containsKey("e")); + assertTrue(aliases.containsKey("f")); + + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.waitForResponses(30L, TimeUnit.SECONDS); + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java b/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java new file mode 100644 index 0000000..652dc40 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/MockNode.java @@ -0,0 +1,12 @@ +package org.xbib.elx.node.test; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; + +public class MockNode extends Node { + + public MockNode(Settings settings) { + super(settings); + } + +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/ReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java similarity index 96% rename from elx-node/src/test/java/org/xbib/elx/node/ReplicaTest.java rename to elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java index 762800f..78a83db 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/ReplicaTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.node; +package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -16,6 +16,8 @@ import org.elasticsearch.index.indexing.IndexingStats; import org.junit.Ignore; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.node.ExtendedNodeClient; +import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.HashMap; import java.util.Map; @@ -26,9 +28,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; @Ignore -public class ReplicaTest extends NodeTestUtils { +public class ReplicaTest extends TestBase { - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName()); @Test public void testReplicaLevel() throws Exception { diff --git a/elx-node/src/test/java/org/xbib/elx/node/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java similarity index 90% rename from elx-node/src/test/java/org/xbib/elx/node/SmokeTest.java rename to elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index cb70fe0..8000063 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.node; +package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,15 +7,17 @@ import org.elasticsearch.common.settings.Settings; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.api.IndexDefinition; +import org.xbib.elx.node.ExtendedNodeClient; +import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -public class SmokeTest extends NodeTestUtils { +public class SmokeTest extends TestBase { - private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); @Test public void smokeTest() throws Exception { @@ -28,7 +30,7 @@ public class SmokeTest extends NodeTestUtils { client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - assertEquals(clusterName, client.getClusterName()); + assertEquals(getClusterName(), client.getClusterName()); client.checkMapping("test"); diff --git a/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java similarity index 78% rename from elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java rename to elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java index 86e30c6..2c486e0 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/NodeTestUtils.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java @@ -1,4 +1,4 @@ -package org.xbib.elx.common; +package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -8,16 +8,17 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.junit.After; import org.junit.Before; -import org.xbib.elx.common.util.NetworkUtils; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -29,54 +30,32 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -public class NodeTestUtils { +public class TestBase { private static final Logger logger = LogManager.getLogger("test"); - private static Random random = new Random(); + private static final Random random = new Random(); - private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); private Map nodes = new HashMap<>(); private Map clients = new HashMap<>(); - private AtomicInteger counter = new AtomicInteger(); - private String cluster; private String host; private int port; - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - - }); - - } - @Before public void startNodes() { try { logger.info("starting"); - setClusterName(); + setClusterName("test-cluster-" + System.getProperty("user.name")); startNode("1"); findNodeAddress(); try { @@ -90,6 +69,12 @@ public class NodeTestUtils { } catch (ElasticsearchTimeoutException e) { throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } catch (Throwable t) { logger.error("startNodes failed", t); } @@ -114,18 +99,15 @@ public class NodeTestUtils { } } - protected void setClusterName() { - this.cluster = "test-helper-cluster-" - + NetworkUtils.getLocalAddress().getHostName() - + "-" + System.getProperty("user.name") - + "-" + counter.incrementAndGet(); + protected void setClusterName(String cluster) { + this.cluster = cluster; } protected String getClusterName() { return cluster; } - protected Settings getSettings() { + protected Settings getTransportSettings() { return settingsBuilder() .put("host", host) .put("port", port) @@ -137,14 +119,14 @@ public class NodeTestUtils { protected Settings getNodeSettings() { return settingsBuilder() .put("cluster.name", cluster) - .put("cluster.routing.schedule", "50ms") - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.zen.multicast.enabled", true) - .put("discovery.zen.multicast.ping_timeout", "5s") - .put("http.enabled", true) - .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) - .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low - .put("index.number_of_replicas", 0) + //.put("cluster.routing.schedule", "50ms") + //.put("cluster.routing.allocation.disk.threshold_enabled", false) + //.put("discovery.zen.multicast.enabled", true) + //.put("discovery.zen.multicast.ping_timeout", "5s") + //.put("http.enabled", true) + //.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + //.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + //.put("index.number_of_replicas", 0) .put("path.home", getHome()) .build(); } @@ -153,30 +135,14 @@ public class NodeTestUtils { return System.getProperty("path.home", System.getProperty("user.dir")); } - public void startNode(String id) { + protected void startNode(String id) { buildNode(id).start(); } - public AbstractClient client(String id) { + protected AbstractClient client(String id) { return clients.get(id); } - private void closeNodes() { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - protected void findNodeAddress() { NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); @@ -210,4 +176,37 @@ public class NodeTestUtils { } return new String(buf); } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } } diff --git a/elx-transport/build.gradle~ b/elx-transport/build.gradle~ deleted file mode 100644 index b47f835..0000000 --- a/elx-transport/build.gradle~ +++ /dev/null @@ -1,63 +0,0 @@ -buildscript { - repositories { - jcenter() - maven { - url 'http://xbib.org/repository' - } - } - dependencies { - classpath "org.xbib.elasticsearch:gradle-plugin-elasticsearch-build:6.2.2.0" - } -} - -apply plugin: 'org.xbib.gradle.plugin.elasticsearch.build' - -configurations { - main - tests -} - -dependencies { - compile project(':common') - testCompile "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" - testRuntime "org.xbib.elasticsearch:elasticsearch-test-framework:${project.property('elasticsearch-devkit.version')}" -} - -jar { - baseName "${rootProject.name}-transport" -} - -task testJar(type: Jar, dependsOn: testClasses) { - baseName = "${project.archivesBaseName}-tests" - from sourceSets.test.output -} - -artifacts { - main jar - tests testJar - archives sourcesJar, javadocJar -} - -esTest { - enabled = true - // test with the jars, not the classes, for security manager - classpath = files(configurations.testRuntime) + configurations.main.artifacts.files + configurations.tests.artifacts.files - systemProperty 'tests.security.manager', 'true' - // maybe we like some extra security policy for our code - systemProperty 'tests.security.policy', '/extra-security.policy' -} -esTest.dependsOn jar, testJar - -randomizedTest { - enabled = false -} - -test { - enabled = false - jvmArgs "-javaagent:" + configurations.alpnagent.asPath - systemProperty 'path.home', projectDir.absolutePath - testLogging { - showStandardStreams = true - exceptionFormat = 'full' - } -} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java index 685b9ec..c50bbd4 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java @@ -57,6 +57,15 @@ public class ExtendedTransportClient extends AbstractExtendedClient { return null; } + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + @Override public ExtendedTransportClient init(Settings settings) throws IOException { super.init(settings); @@ -73,18 +82,6 @@ public class ExtendedTransportClient extends AbstractExtendedClient { return this; } - @Override - public synchronized void close() throws IOException { - super.close(); - logger.info("closing"); - if (getClient() != null) { - TransportClient client = (TransportClient) getClient(); - client.close(); - client.threadPool().shutdown(); - } - logger.info("close completed"); - } - private Collection findAddresses(Settings settings) throws IOException { final int defaultPort = settings.getAsInt("port", 9300); Collection addresses = new ArrayList<>(); diff --git a/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java b/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java deleted file mode 100644 index aad8b8b..0000000 --- a/elx-transport/src/test/java/org/elasticsearch/node/MockNode.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.elasticsearch.node; - -import org.elasticsearch.Version; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.node.internal.InternalSettingsPreparer; -import org.elasticsearch.plugins.Plugin; - -import java.util.ArrayList; -import java.util.Collection; - -public class MockNode extends Node { - - public MockNode() { - super(Settings.EMPTY); - } - - public MockNode(Settings settings) { - super(settings); - } - - public MockNode(Settings settings, Collection> classpathPlugins) { - super(InternalSettingsPreparer.prepareEnvironment(settings, null), Version.CURRENT, classpathPlugins); - } - - public MockNode(Settings settings, Class classpathPlugin) { - this(settings, list(classpathPlugin)); - } - - private static Collection> list(Class classpathPlugin) { - Collection> list = new ArrayList<>(); - list.add(classpathPlugin); - return list; - } -} diff --git a/elx-transport/src/test/java/org/elasticsearch/node/package-info.java b/elx-transport/src/test/java/org/elasticsearch/node/package-info.java deleted file mode 100644 index 8ffed8c..0000000 --- a/elx-transport/src/test/java/org/elasticsearch/node/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.elasticsearch.node; \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java index 56da530..c4dc4fa 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java @@ -28,9 +28,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class ClientTest extends NodeTestUtils { +public class ClientTest extends TestBase { - private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; @@ -50,7 +50,7 @@ public class ClientTest extends NodeTestUtils { public void testClientIndexOp() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); client.newIndex("test"); @@ -69,7 +69,7 @@ public class ClientTest extends NodeTestUtils { public void testSingleDoc() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); @@ -94,7 +94,7 @@ public class ClientTest extends NodeTestUtils { public void testMapping() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); XContentBuilder builder = jsonBuilder() @@ -121,7 +121,7 @@ public class ClientTest extends NodeTestUtils { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); @@ -157,7 +157,7 @@ public class ClientTest extends NodeTestUtils { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java index 6f6b6bd..279fb6b 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java @@ -3,22 +3,23 @@ package org.xbib.elx.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class DuplicateIDTest extends NodeTestUtils { +public class DuplicateIDTest extends TestBase { - private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); + private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); private final static Long MAX_ACTIONS_PER_REQUEST = 1000L; @@ -29,22 +30,24 @@ public class DuplicateIDTest extends NodeTestUtils { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(QueryBuilders.matchAllQuery()); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("test"); + searchRequest.types("test"); + searchRequest.source(builder); + long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java index 7c1fdff..41388c7 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java @@ -2,66 +2,99 @@ package org.xbib.elx.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.metadata.AliasAction; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.Test; +import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -public class IndexShiftTest extends NodeTestUtils { +public class IndexShiftTest extends TestBase { - private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); @Test public void testIndexAlias() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()).build(); + .put(getTransportSettings()).build(); try { - client.newIndex("test1234"); + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + client.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flush(); - client.refreshIndex("test1234"); + client.waitForResponses(30L, TimeUnit.SECONDS); - List simpleAliases = Arrays.asList("a", "b", "c"); - client.shiftIndex("test", "test1234", simpleAliases); + IndexShiftResult indexShiftResult = + client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); - client.newIndex("test5678"); + assertTrue(indexShiftResult.getNewAliases().contains("a")); + assertTrue(indexShiftResult.getNewAliases().contains("b")); + assertTrue(indexShiftResult.getNewAliases().contains("c")); + assertTrue(indexShiftResult.getMovedAliases().isEmpty()); + + Map aliases = client.getAliases("test1234"); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("test")); + + String resolved = client.resolveAlias("test"); + aliases = client.getAliases(resolved); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("test")); + + client.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); } client.flush(); - client.refreshIndex("test5678"); + client.waitForResponses(30L, TimeUnit.SECONDS); - simpleAliases = Arrays.asList("d", "e", "f"); - client.shiftIndex("test", "test5678", simpleAliases, (builder, index, alias) -> - builder.addAlias(index, alias, QueryBuilders.termQuery("my_key", alias))); - Map indexFilters = client.getIndexFilters("test5678"); - logger.info("index filters of index test5678 = {}", indexFilters); - assertTrue(indexFilters.containsKey("a")); - assertTrue(indexFilters.containsKey("b")); - assertTrue(indexFilters.containsKey("c")); - assertTrue(indexFilters.containsKey("d")); - assertTrue(indexFilters.containsKey("e")); + indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, + index, alias).filter(QueryBuilders.termQuery("my_key", alias))) + ); + assertTrue(indexShiftResult.getNewAliases().contains("d")); + assertTrue(indexShiftResult.getNewAliases().contains("e")); + assertTrue(indexShiftResult.getNewAliases().contains("f")); + assertTrue(indexShiftResult.getMovedAliases().contains("a")); + assertTrue(indexShiftResult.getMovedAliases().contains("b")); + assertTrue(indexShiftResult.getMovedAliases().contains("c")); - Map aliases = client.getIndexFilters(client.resolveAlias("test")); - logger.info("aliases of alias test = {}", aliases); + aliases = client.getAliases("test5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); + assertTrue(aliases.containsKey("f")); + + resolved = client.resolveAlias("test"); + aliases = client.getAliases(resolved); + assertTrue(aliases.containsKey("a")); + assertTrue(aliases.containsKey("b")); + assertTrue(aliases.containsKey("c")); + assertTrue(aliases.containsKey("d")); + assertTrue(aliases.containsKey("e")); + assertTrue(aliases.containsKey("f")); - client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java b/elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java new file mode 100644 index 0000000..747e333 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; + +public class MockNode extends Node { + + public MockNode(Settings settings) { + super(settings); + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java index 027b034..c4f9af0 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java @@ -24,9 +24,9 @@ import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -public class ReplicaTest extends NodeTestUtils { +public class ReplicaTest extends TestBase { - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName()); @Test public void testReplicaLevel() throws Exception { @@ -48,7 +48,7 @@ public class ReplicaTest extends NodeTestUtils { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .build(); try { @@ -119,7 +119,7 @@ public class ReplicaTest extends NodeTestUtils { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .build(); Settings settings = Settings.settingsBuilder() diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java index d745015..3721157 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java @@ -3,7 +3,9 @@ package org.xbib.elx.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; import org.junit.Test; +import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; import java.util.concurrent.TimeUnit; @@ -11,25 +13,51 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -public class SmokeTest extends NodeTestUtils { +public class SmokeTest extends TestBase { - private static final Logger logger = LogManager.getLogger(SmokeTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); @Test public void testSingleDocNodeClient() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getSettings()) + .put(getTransportSettings()) .build(); try { client.newIndex("test"); client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); + + assertEquals(getClusterName(), client.getClusterName()); + + client.checkMapping("test"); + + client.update("test", "1", "{ \"name\" : \"Another name\"}"); + client.flush(); + + client.waitForRecovery("test", 10L, TimeUnit.SECONDS); + + client.delete("test", "1"); + client.deleteIndex("test"); + + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test2", Settings.settingsBuilder() + .build()); + assertEquals(0, indexDefinition.getReplicaLevel()); + client.newIndex(indexDefinition); + client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + client.flush(); + client.updateReplicaLevel(indexDefinition, 2); + + int replica = client.getReplicaLevel(indexDefinition); + assertEquals(2, replica); + + client.deleteIndex(indexDefinition); + assertEquals(0, client.getBulkMetric().getFailed().getCount()); + assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); client.close(); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java b/elx-transport/src/test/java/org/xbib/elx/transport/TestBase.java similarity index 78% rename from elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java rename to elx-transport/src/test/java/org/xbib/elx/transport/TestBase.java index 736f87a..95cedb4 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/NodeTestUtils.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/TestBase.java @@ -8,16 +8,17 @@ import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.junit.After; import org.junit.Before; -import org.xbib.elx.common.util.NetworkUtils; import java.io.IOException; import java.nio.file.FileVisitResult; @@ -29,54 +30,32 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -public class NodeTestUtils { +public class TestBase { private static final Logger logger = LogManager.getLogger("test"); - private static Random random = new Random(); + private static final Random random = new Random(); - private static char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); private Map nodes = new HashMap<>(); private Map clients = new HashMap<>(); - private AtomicInteger counter = new AtomicInteger(); - private String cluster; private String host; private int port; - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - - }); - - } - @Before public void startNodes() { try { logger.info("starting"); - setClusterName(); + setClusterName("test-cluster-" + System.getProperty("user.name")); startNode("1"); findNodeAddress(); try { @@ -90,6 +69,12 @@ public class NodeTestUtils { } catch (ElasticsearchTimeoutException e) { throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } catch (Throwable t) { logger.error("startNodes failed", t); } @@ -114,18 +99,15 @@ public class NodeTestUtils { } } - protected void setClusterName() { - this.cluster = "test-helper-cluster-" - + NetworkUtils.getLocalAddress().getHostName() - + "-" + System.getProperty("user.name") - + "-" + counter.incrementAndGet(); + protected void setClusterName(String cluster) { + this.cluster = cluster; } protected String getClusterName() { return cluster; } - protected Settings getSettings() { + protected Settings getTransportSettings() { return settingsBuilder() .put("host", host) .put("port", port) @@ -137,14 +119,14 @@ public class NodeTestUtils { protected Settings getNodeSettings() { return settingsBuilder() .put("cluster.name", cluster) - .put("cluster.routing.schedule", "50ms") - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.zen.multicast.enabled", true) - .put("discovery.zen.multicast.ping_timeout", "5s") - .put("http.enabled", true) - .put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) - .put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low - .put("index.number_of_replicas", 0) + //.put("cluster.routing.schedule", "50ms") + //.put("cluster.routing.allocation.disk.threshold_enabled", false) + //.put("discovery.zen.multicast.enabled", true) + //.put("discovery.zen.multicast.ping_timeout", "5s") + //.put("http.enabled", true) + //.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) + //.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low + //.put("index.number_of_replicas", 0) .put("path.home", getHome()) .build(); } @@ -153,30 +135,14 @@ public class NodeTestUtils { return System.getProperty("path.home", System.getProperty("user.dir")); } - public void startNode(String id) { + protected void startNode(String id) { buildNode(id).start(); } - public AbstractClient client(String id) { + protected AbstractClient client(String id) { return clients.get(id); } - private void closeNodes() { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - protected void findNodeAddress() { NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); @@ -194,7 +160,6 @@ public class NodeTestUtils { .put(getNodeSettings()) .put("name", id) .build(); - logger.info("settings={}", nodeSettings.getAsMap()); Node node = new MockNode(nodeSettings); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); @@ -211,4 +176,37 @@ public class NodeTestUtils { } return new String(buf); } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + private static void deleteFiles() throws IOException { + Path directory = Paths.get(getHome() + "/data"); + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } } diff --git a/elx-transport/src/test/resources/log4j2.xml b/elx-transport/src/test/resources/log4j2.xml index 6c323f8..1258d7f 100644 --- a/elx-transport/src/test/resources/log4j2.xml +++ b/elx-transport/src/test/resources/log4j2.xml @@ -6,7 +6,7 @@ - + diff --git a/gradle.properties b/gradle.properties index 98d19b7..7317e63 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.5 +version = 2.2.1.6 xbib-metrics.version = 1.1.0 xbib-guice.version = 4.0.4 diff --git a/gradle/publish.gradle~ b/gradle/publish.gradle~ deleted file mode 100644 index e04b20b..0000000 --- a/gradle/publish.gradle~ +++ /dev/null @@ -1,104 +0,0 @@ - -task xbibUpload(type: Upload) { - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty("xbibUsername")) { - mavenDeployer { - configuration = configurations.wagon - repository(url: 'scpexe://xbib.org/repository') { - authentication(userName: xbibUsername, privateKey: xbibPrivateKey) - } - } - } - } -} - -task sonaTypeUpload(type: Upload) { - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty('ossrhUsername')) { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - pom.project { - name name - description description - packaging 'jar' - inceptionYear '2012' - url scmUrl - organization { - name 'xbib' - url 'http://xbib.org' - } - developers { - developer { - id user - name 'Jörg Prante' - email 'joergprante@gmail.com' - url 'https://github.com/jprante' - } - } - scm { - url scmUrl - connection scmConnection - developerConnection scmDeveloperConnection - } - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - } - } - } - } -} - - -task hbzUpload(type: Upload) { - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty('hbzUserName')) { - mavenDeployer { - configuration = configurations.wagon - beforeDeployment { MavenDeployment deployment -> - signing.signPom(deployment) - } - repository(url: uri(hbzUrl)) { - authentication(userName: hbzUserName, privateKey: hbzPrivateKey) - } - pom.project { - developers { - developer { - id 'jprante' - name 'Jörg Prante' - email 'joergprante@gmail.com' - url 'https://github.com/jprante' - } - } - scm { - url 'https://github.com/xbib/elasticsearch-webapp-libraryservice' - connection 'scm:git:git://github.com/xbib/elasticsaerch-webapp-libraryservice.git' - developerConnection 'scm:git:git://github.com/xbib/elasticsaerch-webapp-libraryservice.git' - } - inceptionYear '2016' - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - } - } - } - } -} From 22ab37b3e534d93fd18995ef33feefa2e6d6505b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sat, 23 Feb 2019 13:38:41 +0100 Subject: [PATCH 02/19] metric 1.2.0, fix shift index, add prune index test --- build.gradle | 15 +++- elx-api/build.gradle | 2 +- .../java/org/xbib/elx/api/BulkMetric.java | 4 +- .../java/org/xbib/elx/api/BulkProcessor.java | 4 +- .../elx/common/AbstractExtendedClient.java | 54 +++++++------ .../xbib/elx/common/DefaultBulkMetric.java | 8 +- .../xbib/elx/common/DefaultBulkProcessor.java | 2 + .../xbib/elx/node/test/IndexPruneTest.java | 80 +++++++++++++++++++ .../xbib/elx/node/test/IndexShiftTest.java | 1 - elx-node/src/test/resources/log4j2.xml | 2 +- .../xbib/elx/transport/IndexPruneTest.java | 79 ++++++++++++++++++ elx-transport/src/test/resources/log4j2.xml | 2 +- gradle.properties | 2 +- 13 files changed, 215 insertions(+), 40 deletions(-) create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java diff --git a/build.gradle b/build.gradle index 090181a..2e44f5f 100644 --- a/build.gradle +++ b/build.gradle @@ -41,11 +41,19 @@ subprojects { testCompile "junit:junit:${project.property('junit.version')}" testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" + asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" } - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + compileJava { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + compileTestJava { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } tasks.withType(JavaCompile) { options.compilerArgs << "-Xlint:all" @@ -57,6 +65,7 @@ subprojects { test { jvmArgs =[ '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', '--add-opens=java.base/java.nio=ALL-UNNAMED' ] systemProperty 'jna.debug_load', 'true' @@ -78,7 +87,7 @@ subprojects { options.overview = "src/docs/asciidoclet/overview.adoc" options.addStringOption "-base-dir", "${projectDir}" options.addStringOption "-attribute", - "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" + "name=${project.name},version=${project.version},title-link=https://github.com/jprante/${project.name}" configure(options) { noTimestamp = true } diff --git a/elx-api/build.gradle b/elx-api/build.gradle index 9e7b7be..47596cb 100644 --- a/elx-api/build.gradle +++ b/elx-api/build.gradle @@ -1,5 +1,5 @@ dependencies { - compile "org.xbib:metrics:${project.property('xbib-metrics.version')}" + compile "org.xbib:metrics-common:${project.property('xbib-metrics.version')}" compile("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") { // exclude ES jackson yaml, cbor, smile versions exclude group: 'com.fasterxml.jackson.dataformat' diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java index 3a406fb..af825e5 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java @@ -1,8 +1,8 @@ package org.xbib.elx.api; import org.elasticsearch.common.settings.Settings; -import org.xbib.metrics.Count; -import org.xbib.metrics.Metered; +import org.xbib.metrics.api.Count; +import org.xbib.metrics.api.Metered; import java.io.Closeable; diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java index 2703acb..52b6d04 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -10,10 +10,10 @@ import java.util.concurrent.TimeUnit; public interface BulkProcessor extends Closeable, Flushable { - @SuppressWarnings("rawtype") + @SuppressWarnings("rawtypes") BulkProcessor add(ActionRequest request); - @SuppressWarnings("rawtype") + @SuppressWarnings("rawtypes") BulkProcessor add(ActionRequest request, Object payload); boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException; diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 5ea81ae..4b97f4a 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -309,7 +309,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { + public ExtendedClient newIndex(String index, Settings settings, Map mapping) { ensureActive(); if (index == null) { logger.warn("no index name given to create index"); @@ -378,18 +378,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return this; } - @Override - public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) - .source(source)); - } - @Override public ExtendedClient index(String index, String id, boolean create, String source) { return index(new IndexRequest(index, TYPE_NAME, id).create(create) .source(source.getBytes(StandardCharsets.UTF_8))); } + @Override + public ExtendedClient index(String index, String id, boolean create, BytesReference source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source)); + } + @Override public ExtendedClient index(IndexRequest indexRequest) { ensureActive(); @@ -411,12 +411,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient update(String index, String id, BytesReference source) { - return update(new UpdateRequest(index, TYPE_NAME, id).doc(source)); + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source)); } @Override public ExtendedClient update(String index, String id, String source) { - return update(new UpdateRequest(index, TYPE_NAME, id).doc(source.getBytes(StandardCharsets.UTF_8))); + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source.getBytes(StandardCharsets.UTF_8))); } @Override @@ -439,18 +441,22 @@ public abstract class AbstractExtendedClient implements ExtendedClient { RecoveryRequest recoveryRequest = new RecoveryRequest(); recoveryRequest.indices(index); recoveryRequest.activeOnly(true); - RecoveryResponse response = client.execute(RecoveryAction.INSTANCE, recoveryRequest).actionGet(); - int shards = response.getTotalShards(); - TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() - .indices(new String[]{index}) - .waitForActiveShards(shards) - .timeout(timeout); - ClusterHealthResponse healthResponse = - client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - logger.error("timeout waiting for recovery"); - return false; + RecoveryResponse recoveryResponse = client.execute(RecoveryAction.INSTANCE, recoveryRequest).actionGet(); + if (recoveryResponse.hasRecoveries()) { + int shards = recoveryResponse.getTotalShards(); + logger.info("shards = {}", shards); + logger.info(recoveryResponse.shardRecoveryStates()); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() + .indices(new String[]{index}) + .waitForActiveShards(shards) + .timeout(timeout); + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + logger.error("timeout waiting for recovery"); + return false; + } } return true; } @@ -641,10 +647,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { oldIndex, alias)); if (filter != null) { indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, index).filter(filter)); + fullIndexName, alias).filter(filter)); } else { indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, index)); + fullIndexName, alias)); } moveAliases.add(alias); } @@ -733,7 +739,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (m2.matches()) { Integer i2 = Integer.parseInt(m2.group(2)); int kept = candidateIndices.size() - indicesToDelete.size(); - if ((delta == 0 || (delta > 0 && i1 - i2 > delta)) && mintokeep <= kept) { + if ((delta == 0 || (delta > 0 && i1 - i2 >= delta)) && mintokeep <= kept) { indicesToDelete.add(s); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java index a956c4d..8127e29 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java @@ -2,10 +2,10 @@ package org.xbib.elx.common; import org.elasticsearch.common.settings.Settings; import org.xbib.elx.api.BulkMetric; -import org.xbib.metrics.Count; -import org.xbib.metrics.CountMetric; -import org.xbib.metrics.Meter; -import org.xbib.metrics.Metered; +import org.xbib.metrics.api.Count; +import org.xbib.metrics.api.Metered; +import org.xbib.metrics.common.CountMetric; +import org.xbib.metrics.common.Meter; import java.util.concurrent.Executors; diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 224f507..436d375 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -132,6 +132,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @param request request * @return his bulk processor */ + @SuppressWarnings("rawtypes") @Override public DefaultBulkProcessor add(ActionRequest request) { return add(request, null); @@ -144,6 +145,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @param payload payload * @return his bulk processor */ + @SuppressWarnings("rawtypes") @Override public DefaultBulkProcessor add(ActionRequest request, Object payload) { internalAdd(request, payload); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java new file mode 100644 index 0000000..b0b6428 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -0,0 +1,80 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.junit.Test; +import org.xbib.elx.api.IndexPruneResult; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.node.ExtendedNodeClient; +import org.xbib.elx.node.ExtendedNodeClientProvider; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class IndexPruneTest extends TestBase { + + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + + @Test + public void testPrune() throws IOException { + final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + .provider(ExtendedNodeClientProvider.class) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + client.newIndex("test1", settings); + client.shiftIndex("test", "test1", Collections.emptyList()); + client.newIndex("test2", settings); + client.shiftIndex("test", "test2", Collections.emptyList()); + client.newIndex("test3", settings); + client.shiftIndex("test", "test3", Collections.emptyList()); + client.newIndex("test4", settings); + client.shiftIndex("test", "test4", Collections.emptyList()); + + IndexPruneResult indexPruneResult = + client.pruneIndex("test", "test4", 2, 2, true); + + assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); + assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); + + List list = new ArrayList<>(); + for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { + IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); + indicesExistsRequest.indices(new String[] { index }); + IndicesExistsResponse indicesExistsResponse = + client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + list.add(indicesExistsResponse.isExists()); + } + logger.info(list); + assertFalse(list.get(0)); + assertFalse(list.get(1)); + assertTrue(list.get(2)); + assertTrue(list.get(3)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index 8b31d89..6c900e8 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -100,7 +100,6 @@ public class IndexShiftTest extends TestBase { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.waitForResponses(30L, TimeUnit.SECONDS); client.close(); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); diff --git a/elx-node/src/test/resources/log4j2.xml b/elx-node/src/test/resources/log4j2.xml index 1258d7f..6c323f8 100644 --- a/elx-node/src/test/resources/log4j2.xml +++ b/elx-node/src/test/resources/log4j2.xml @@ -6,7 +6,7 @@ - + diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java new file mode 100644 index 0000000..4ce1843 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java @@ -0,0 +1,79 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.junit.Test; +import org.xbib.elx.api.IndexPruneResult; +import org.xbib.elx.common.ClientBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class IndexPruneTest extends TestBase { + + private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + + @Test + public void testPrune() throws IOException { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(getTransportSettings()) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + client.newIndex("test1", settings); + client.shiftIndex("test", "test1", Collections.emptyList()); + client.newIndex("test2", settings); + client.shiftIndex("test", "test2", Collections.emptyList()); + client.newIndex("test3", settings); + client.shiftIndex("test", "test3", Collections.emptyList()); + client.newIndex("test4", settings); + client.shiftIndex("test", "test4", Collections.emptyList()); + + IndexPruneResult indexPruneResult = + client.pruneIndex("test", "test4", 2, 2, true); + + assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); + assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); + + List list = new ArrayList<>(); + for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { + IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); + indicesExistsRequest.indices(new String[] { index }); + IndicesExistsResponse indicesExistsResponse = + client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + list.add(indicesExistsResponse.isExists()); + } + logger.info(list); + assertFalse(list.get(0)); + assertFalse(list.get(1)); + assertTrue(list.get(2)); + assertTrue(list.get(3)); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } finally { + client.close(); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + } + } +} diff --git a/elx-transport/src/test/resources/log4j2.xml b/elx-transport/src/test/resources/log4j2.xml index 1258d7f..6c323f8 100644 --- a/elx-transport/src/test/resources/log4j2.xml +++ b/elx-transport/src/test/resources/log4j2.xml @@ -6,7 +6,7 @@ - + diff --git a/gradle.properties b/gradle.properties index 7317e63..ea165e1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,7 +2,7 @@ group = org.xbib name = elx version = 2.2.1.6 -xbib-metrics.version = 1.1.0 +xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 elasticsearch.version = 2.2.1 From 41b18f4ae7b497c98ca8e7ec754a6abf1dcd6289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sat, 23 Feb 2019 13:51:57 +0100 Subject: [PATCH 03/19] set travis to openjdk11 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a830350..0c75adf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: java sudo: required jdk: - - oraclejdk8 + - openjdk11 cache: directories: - $HOME/.m2 From 37b664f1b61104db061a2787fc08499e31087dec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sat, 23 Feb 2019 15:26:36 +0100 Subject: [PATCH 04/19] fix wait for active shards --- .../org/xbib/elx/common/AbstractExtendedClient.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 4b97f4a..586013b 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -438,14 +438,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { ensureActive(); ensureIndexGiven(index); - RecoveryRequest recoveryRequest = new RecoveryRequest(); - recoveryRequest.indices(index); - recoveryRequest.activeOnly(true); - RecoveryResponse recoveryResponse = client.execute(RecoveryAction.INSTANCE, recoveryRequest).actionGet(); - if (recoveryResponse.hasRecoveries()) { - int shards = recoveryResponse.getTotalShards(); - logger.info("shards = {}", shards); - logger.info(recoveryResponse.shardRecoveryStates()); + GetSettingsRequest settingsRequest = new GetSettingsRequest(); + settingsRequest.indices(index); + GetSettingsResponse settingsResponse = client.execute(GetSettingsAction.INSTANCE, settingsRequest).actionGet(); + int shards = settingsResponse.getIndexToSettings().get(index).getAsInt("index.number_of_shards", -1); + if (shards > 0) { TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() .indices(new String[]{index}) From 00d3ef984ce8273358aeee154a9532862df4c1a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sat, 23 Feb 2019 17:37:14 +0100 Subject: [PATCH 05/19] clean up --- .../main/java/org/xbib/elx/common/AbstractExtendedClient.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index 586013b..de16805 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -33,9 +33,6 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.recovery.RecoveryAction; -import org.elasticsearch.action.admin.indices.recovery.RecoveryRequest; -import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; From 47a0ca64fb5eda784b5546907b8ff6160db11053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sat, 23 Feb 2019 23:08:21 +0100 Subject: [PATCH 06/19] 2.2.1.7 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index ea165e1..ae3020f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.6 +version = 2.2.1.7 xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 From 3439386e026d16f254d5260a3655006580f4d54f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 2 May 2019 19:46:21 +0200 Subject: [PATCH 07/19] switch to Junit 5 --- build.gradle | 58 +- elx-api/build.gradle | 11 +- .../java/org/xbib/elx/api/ReadClient.java | 25 + .../org/xbib/elx/api/ReadClientProvider.java | 7 + .../elx/common/DefaultBulkController.java | 20 +- .../xbib/elx/common/DefaultBulkProcessor.java | 24 +- .../org/xbib/elx/common/test/AliasTest.java | 58 +- .../elx/common/test/ClusterBlockTest.java | 48 -- .../test/MockExtendedClientProviderTest.java | 8 +- .../org/xbib/elx/common/test/NetworkTest.java | 10 +- .../org/xbib/elx/common/test/SearchTest.java | 51 +- .../org/xbib/elx/common/test/SimpleTest.java | 28 +- .../org/xbib/elx/common/test/TestBase.java | 212 ------- .../xbib/elx/common/test/TestExtension.java | 216 +++++++ .../xbib/elx/common/test/WildcardTest.java | 62 +- elx-http/build.gradle | 4 + .../org/xbib/elx/http/ExtendedHttpClient.java | 127 ++++ .../elx/http/ExtendedHttpClientProvider.java | 10 + .../java/org/xbib/elx/http/HttpAction.java | 169 +++++ .../org/xbib/elx/http/HttpActionContext.java | 60 ++ .../elx/http/action/get/HttpGetAction.java | 179 ++++++ .../http/action/get/HttpMultiGetAction.java | 255 ++++++++ .../http/action/search/HttpSearchAction.java | 597 ++++++++++++++++++ .../elx/http/util/AbstractObjectParser.java | 217 +++++++ .../xbib/elx/http/util/CheckedBiConsumer.java | 11 + .../xbib/elx/http/util/CheckedFunction.java | 6 + .../org/xbib/elx/http/util/ContextParser.java | 13 + .../util/NamedObjectNotFoundException.java | 14 + .../elx/http/util/NamedXContentRegistry.java | 101 +++ .../org/xbib/elx/http/util/ObjectParser.java | 441 +++++++++++++ .../elx/http/util/XContentParseException.java | 47 ++ .../elx/http/util/XContentParserUtils.java | 68 ++ .../http/util/aggregations/CommonFields.java | 18 + .../util/aggregations/ParsedAggregation.java | 40 ++ .../ParsedMultiBucketAggregation.java | 149 +++++ .../util/aggregations/ParsedStringTerms.java | 103 +++ .../http/util/aggregations/ParsedTerms.java | 118 ++++ .../org.xbib.elx.api.ExtendedClientProvider | 1 + .../services/org.xbib.elx.http.HttpAction | 3 + .../org/xbib/elx/http/test/ClientTest.java | 122 ++++ .../org/xbib/elx/http/test}/MockNode.java | 3 +- .../org/xbib/elx/http/test}/TestBase.java | 38 +- elx-http/src/test/resources/log4j2.xml | 13 + .../org/xbib/elx/node/test/ClientTest.java | 159 ++--- .../xbib/elx/node/test/DuplicateIDTest.java | 36 +- .../xbib/elx/node/test/IndexPruneTest.java | 53 +- .../xbib/elx/node/test/IndexShiftTest.java | 40 +- .../org/xbib/elx/node/test/ReplicaTest.java | 151 ----- .../org/xbib/elx/node/test/SmokeTest.java | 46 +- .../java/org/xbib/elx/node/test/TestBase.java | 212 ------- .../org/xbib/elx/node/test/TestExtension.java | 213 +++++++ .../org/xbib/elx/transport/ReplicaTest.java | 150 ----- .../org/xbib/elx/transport/package-info.java | 1 - .../elx/transport/{ => test}/ClientTest.java | 168 +++-- .../transport/{ => test}/DuplicateIDTest.java | 42 +- .../transport/{ => test}/IndexPruneTest.java | 54 +- .../transport/{ => test}/IndexShiftTest.java | 44 +- .../org/xbib/elx/transport/test/MockNode.java | 11 + .../elx/transport/{ => test}/SmokeTest.java | 47 +- .../elx/transport/test/TestExtension.java | 229 +++++++ .../xbib/elx/transport/test/package-info.java | 1 + gradle.properties | 7 +- gradle/wrapper/gradle-wrapper.jar | Bin 55190 -> 55616 bytes gradle/wrapper/gradle-wrapper.properties | 4 +- gradlew | 18 +- gradlew.bat | 18 +- settings.gradle | 1 - 67 files changed, 4210 insertions(+), 1260 deletions(-) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/ReadClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java delete mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java delete mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java create mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java create mode 100644 elx-http/build.gradle create mode 100644 elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpAction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/AbstractObjectParser.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/CheckedBiConsumer.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/CheckedFunction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/ContextParser.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/NamedObjectNotFoundException.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/NamedXContentRegistry.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/ObjectParser.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/XContentParseException.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/XContentParserUtils.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/aggregations/CommonFields.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedAggregation.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedMultiBucketAggregation.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedStringTerms.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedTerms.java create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction create mode 100644 elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java rename {elx-transport/src/test/java/org/xbib/elx/transport => elx-http/src/test/java/org/xbib/elx/http/test}/MockNode.java (85%) rename {elx-transport/src/test/java/org/xbib/elx/transport => elx-http/src/test/java/org/xbib/elx/http/test}/TestBase.java (86%) create mode 100644 elx-http/src/test/resources/log4j2.xml delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java delete mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java delete mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/package-info.java rename elx-transport/src/test/java/org/xbib/elx/transport/{ => test}/ClientTest.java (67%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ => test}/DuplicateIDTest.java (60%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ => test}/IndexPruneTest.java (62%) rename elx-transport/src/test/java/org/xbib/elx/transport/{ => test}/IndexShiftTest.java (73%) create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java rename elx-transport/src/test/java/org/xbib/elx/transport/{ => test}/SmokeTest.java (61%) create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/package-info.java diff --git a/build.gradle b/build.gradle index 2e44f5f..1333813 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s JavaVersion.current() if (JavaVersion.current() < JavaVersion.VERSION_11) { - throw new GradleException("This build must be run with java 11 or higher") + throw new GradleException("The build must be run with Java 11") } subprojects { @@ -38,9 +38,11 @@ subprojects { } dependencies { - testCompile "junit:junit:${project.property('junit.version')}" - testCompile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" - testCompile "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" + testImplementation "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" + testImplementation "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}" + testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" } @@ -63,43 +65,44 @@ subprojects { } test { - jvmArgs =[ - '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', - '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED' - ] + useJUnitPlatform() + // we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery + doFirst { + jvmArgs = [ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] + } + systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' systemProperty 'jna.debug_load', 'true' + systemProperty 'path.home', "${project.buildDir}" + failFast = false testLogging { - showStandardStreams = true - exceptionFormat = 'full' + events 'PASSED', 'FAILED', 'SKIPPED' + } + afterSuite { desc, result -> + if (!desc.parent) { + println "\nTest result: ${result.resultType}" + println "Test summary: ${result.testCount} tests, " + + "${result.successfulTestCount} succeeded, " + + "${result.failedTestCount} failed, " + + "${result.skippedTestCount} skipped" + } } } clean { - delete "data" - delete "logs" delete "out" } - /*javadoc { - options.docletpath = configurations.asciidoclet.files.asType(List) - options.doclet = 'org.asciidoctor.Asciidoclet' - options.overview = "src/docs/asciidoclet/overview.adoc" - options.addStringOption "-base-dir", "${projectDir}" - options.addStringOption "-attribute", - "name=${project.name},version=${project.version},title-link=https://github.com/jprante/${project.name}" - configure(options) { - noTimestamp = true - } - }*/ - task javadocJar(type: Jar, dependsOn: javadoc) { - classifier 'javadoc' + archiveClassifier.set('javadoc') } task sourcesJar(type: Jar, dependsOn: classes) { from sourceSets.main.allSource - classifier 'sources' + archiveClassifier.set('sources') } artifacts { @@ -135,6 +138,7 @@ subprojects { html.enabled = true } } + tasks.withType(Checkstyle) { ignoreFailures = true reports { diff --git a/elx-api/build.gradle b/elx-api/build.gradle index 47596cb..21f7734 100644 --- a/elx-api/build.gradle +++ b/elx-api/build.gradle @@ -1,19 +1,20 @@ dependencies { compile "org.xbib:metrics-common:${project.property('xbib-metrics.version')}" compile("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") { - // exclude ES jackson yaml, cbor, smile versions + // exclude original ES jackson yaml, cbor, smile version (2.6.2) exclude group: 'com.fasterxml.jackson.dataformat' - // dependencies that are not meant for client + // these dependencies that are not meant for client applications exclude module: 'securesm' // we use log4j2, not log4j exclude group: 'log4j' } // override log4j2 of Elastic with ours compile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + // override ES jackson with our jackson version // for Elasticsearch session, ES uses SMILE when encoding source for SearchRequest - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson-dataformat.version')}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson.version')}" // CBOR ist default JSON content compression encoding in ES 2.2.1 - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson-dataformat.version')}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson.version')}" // not used, but maybe in other projects - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson-dataformat.version')}" + compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson.version')}" } \ No newline at end of file diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java new file mode 100644 index 0000000..2ecb857 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java @@ -0,0 +1,25 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; + +public interface ReadClient { + + ActionFuture get(GetRequest getRequest); + + void get(GetRequest request, ActionListener listener); + + ActionFuture multiGet(MultiGetRequest request); + + void multiGet(MultiGetRequest request, ActionListener listener); + + ActionFuture search(SearchRequest request); + + void search(SearchRequest request, ActionListener listener); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java new file mode 100644 index 0000000..6640686 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface ReadClientProvider { + + C getReadClient(); +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index 30d5b52..f6af929 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -83,12 +82,12 @@ public class DefaultBulkController implements BulkController { maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); } this.bulkListener = new BulkListener(); - DefaultBulkProcessor.Builder builder = DefaultBulkProcessor.builder((Client) client.getClient(), bulkListener) + this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) .setConcurrentRequests(maxConcurrentRequests) .setFlushInterval(flushIngestInterval) - .setBulkSize(maxVolumePerRequest); - this.bulkProcessor = builder.build(); + .setBulkSize(maxVolumePerRequest) + .build(); this.active.set(true); } @@ -115,6 +114,7 @@ public class DefaultBulkController implements BulkController { @Override public void index(IndexRequest indexRequest) { + ensureActiveAndBulk(); if (!active.get()) { throw new IllegalStateException("inactive"); } @@ -226,6 +226,18 @@ public class DefaultBulkController implements BulkController { } } + private void ensureActiveAndBulk() { + if (!active.get()) { + throw new IllegalStateException("inactive"); + } + if (bulkProcessor == null) { + throw new UnsupportedOperationException("bulk processor not present"); + } + if (bulkListener == null) { + throw new UnsupportedOperationException("bulk listener not present"); + } + } + private class BulkListener implements DefaultBulkProcessor.Listener { private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 436d375..e8048cc 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -5,7 +5,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -44,20 +44,22 @@ public class DefaultBulkProcessor implements BulkProcessor { private volatile boolean closed; - private DefaultBulkProcessor(Client client, Listener listener, String name, int concurrentRequests, + private DefaultBulkProcessor(ElasticsearchClient client, Listener listener, String name, int concurrentRequests, int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) { this.executionIdGen = new AtomicLong(); this.closed = false; this.bulkActions = bulkActions; this.bulkSize = bulkSize.getBytes(); this.bulkRequest = new BulkRequest(); + if (listener == null) { + throw new IllegalArgumentException(); + } this.bulkRequestHandler = concurrentRequests == 0 ? new SyncBulkRequestHandler(client, listener) : new AsyncBulkRequestHandler(client, listener, concurrentRequests); if (flushInterval != null) { this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, - EsExecutors.daemonThreadFactory(client.settings(), - name != null ? "[" + name + "]" : "" + "bulk_processor")); + EsExecutors.daemonThreadFactory(name != null ? "[" + name + "]" : "" + "bulk_processor")); this.scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); this.scheduler.setContinueExistingPeriodicTasksAfterShutdownPolicy(false); this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(new Flush(), flushInterval.millis(), @@ -68,7 +70,7 @@ public class DefaultBulkProcessor implements BulkProcessor { } } - public static Builder builder(Client client, Listener listener) { + public static Builder builder(ElasticsearchClient client, Listener listener) { if (client == null) { throw new NullPointerException("The client you specified while building a BulkProcessor is null"); } @@ -215,7 +217,7 @@ public class DefaultBulkProcessor implements BulkProcessor { */ public static class Builder { - private final Client client; + private final ElasticsearchClient client; private final Listener listener; @@ -236,7 +238,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @param client the client * @param listener the listener */ - Builder(Client client, Listener listener) { + Builder(ElasticsearchClient client, Listener listener) { this.client = client; this.listener = listener; } @@ -330,11 +332,11 @@ public class DefaultBulkProcessor implements BulkProcessor { private static class SyncBulkRequestHandler implements BulkRequestHandler { - private final Client client; + private final ElasticsearchClient client; private final DefaultBulkProcessor.Listener listener; - SyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener) { + SyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener) { this.client = client; this.listener = listener; } @@ -362,7 +364,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private static class AsyncBulkRequestHandler implements BulkRequestHandler { - private final Client client; + private final ElasticsearchClient client; private final DefaultBulkProcessor.Listener listener; @@ -370,7 +372,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private final int concurrentRequests; - private AsyncBulkRequestHandler(Client client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { + private AsyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { this.client = client; this.listener = listener; this.concurrentRequests = concurrentRequests; diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java index b1b7f95..928dbe6 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java @@ -1,20 +1,20 @@ package org.xbib.elx.common.test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesAction; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.Strings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.util.Collections; import java.util.Iterator; @@ -23,53 +23,58 @@ import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** - * - */ -public class AliasTest extends TestBase { +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class AliasTest { private static final Logger logger = LogManager.getLogger(AliasTest.class.getName()); + private final TestExtension.Helper helper; + + AliasTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testAlias() { - Client client = client("1"); - CreateIndexRequest indexRequest = new CreateIndexRequest("test"); - client.admin().indices().create(indexRequest).actionGet(); - // put alias + void testAlias() { + ElasticsearchClient client = helper.client("1"); + CreateIndexRequest indexRequest = new CreateIndexRequest("test_index"); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); - String[] indices = new String[]{"test"}; + String[] indices = new String[]{"test_index"}; String[] aliases = new String[]{"test_alias"}; IndicesAliasesRequest.AliasActions aliasAction = new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases); indicesAliasesRequest.addAliasAction(aliasAction); - client.admin().indices().aliases(indicesAliasesRequest).actionGet(); + client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); // get alias GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY); long t0 = System.nanoTime(); - GetAliasesResponse getAliasesResponse = client.admin().indices().getAliases(getAliasesRequest).actionGet(); + GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); long t1 = (System.nanoTime() - t0) / 1000000; logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1); assertTrue(t1 >= 0); } @Test - public void testMostRecentIndex() { - Client client = client("1"); + void testMostRecentIndex() { + ElasticsearchClient client = helper.client("1"); String alias = "test"; CreateIndexRequest indexRequest = new CreateIndexRequest("test20160101"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); indexRequest = new CreateIndexRequest("test20160102"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); indexRequest = new CreateIndexRequest("test20160103"); - client.admin().indices().create(indexRequest).actionGet(); + client.execute(CreateIndexAction.INSTANCE, indexRequest).actionGet(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); - String[] indices = new String[]{"test20160101", "test20160102", "test20160103"}; - String[] aliases = new String[]{alias}; + String[] indices = new String[] { "test20160101", "test20160102", "test20160103" }; + String[] aliases = new String[] { alias }; IndicesAliasesRequest.AliasActions aliasAction = new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, indices, aliases); indicesAliasesRequest.addAliasAction(aliasAction); - client.admin().indices().aliases(indicesAliasesRequest).actionGet(); - + client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); GetAliasesRequest getAliasesRequest = new GetAliasesRequest(); getAliasesRequest.aliases(alias); GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); @@ -89,5 +94,4 @@ public class AliasTest extends TestBase { assertEquals("test20160101", it.next()); logger.info("success: result={}", result); } - } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java deleted file mode 100644 index dae869f..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/ClusterBlockTest.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.xbib.elx.common.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.junit.Before; -import org.junit.Test; - -public class ClusterBlockTest extends TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - @Before - public void startNodes() { - try { - setClusterName("test-cluster" + System.getProperty("user.name")); - startNode("1"); - // do not wait for green health state - logger.info("ready"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } - } - - @Override - protected Settings getNodeSettings() { - return Settings.settingsBuilder() - .put(super.getNodeSettings()) - .put("discovery.zen.minimum_master_nodes", 2) // block until we have two nodes - .build(); - } - - @Test(expected = ClusterBlockException.class) - public void testClusterBlock() throws Exception { - Client client = client("1"); - XContentBuilder builder = XContentFactory.jsonBuilder().startObject().field("field1", "value1").endObject(); - IndexRequestBuilder irb = client.prepareIndex("test", "test", "1").setSource(builder); - BulkRequestBuilder brb = client.prepareBulk(); - brb.add(irb); - brb.execute().actionGet(); - } -} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java index cbe7972..e21817c 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java @@ -1,18 +1,18 @@ package org.xbib.elx.common.test; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.MockExtendedClient; import org.xbib.elx.common.MockExtendedClientProvider; import java.io.IOException; -import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotNull; -public class MockExtendedClientProviderTest { +class MockExtendedClientProviderTest { @Test - public void testMockExtendedProvider() throws IOException { + void testMockExtendedProvider() throws IOException { MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); assertNotNull(client); } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java index 7933343..ef44d74 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/NetworkTest.java @@ -2,20 +2,22 @@ package org.xbib.elx.common.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import java.net.InetAddress; import java.net.NetworkInterface; import java.util.Collections; import java.util.Enumeration; -public class NetworkTest { +// walk over all found interfaces (this is slow - multicast/pings are performed) +@Disabled +class NetworkTest { private static final Logger logger = LogManager.getLogger(NetworkTest.class); @Test - public void testNetwork() throws Exception { - // walk over all found interfaces (this is slow - multicast/pings are performed) + void testNetwork() throws Exception { Enumeration nets = NetworkInterface.getNetworkInterfaces(); for (NetworkInterface netint : Collections.list(nets)) { System.out.println("checking network interface = " + netint.getName()); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index 6e23f0b..2b82483 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -1,27 +1,38 @@ package org.xbib.elx.common.test; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortOrder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class SearchTest extends TestBase { +@ExtendWith(TestExtension.class) +class SearchTest { + + private final TestExtension.Helper helper; + + SearchTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testSearch() throws Exception { - Client client = client("1"); + void testSearch() throws Exception { + ElasticsearchClient client = helper.client("1"); BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 1; i++) { IndexRequest indexRequest = new IndexRequest("pages", "row") .source(XContentFactory.jsonBuilder() .startObject() @@ -39,18 +50,20 @@ public class SearchTest extends TestBase { .endObject()); builder.add(indexRequest); } - client.bulk(builder.request()).actionGet(); - client.admin().indices().refresh(new RefreshRequest()).actionGet(); - - for (int i = 0; i < 100; i++) { + client.execute(BulkAction.INSTANCE, builder.request()).actionGet(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); + for (int i = 0; i < 1; i++) { QueryBuilder queryStringBuilder = QueryBuilders.queryStringQuery("rs:" + 1234); - SearchRequestBuilder requestBuilder = client.prepareSearch() - .setIndices("pages") - .setTypes("row") - .setQuery(queryStringBuilder) - .addSort("rowcount", SortOrder.DESC) - .setFrom(i * 10).setSize(10); - SearchResponse searchResponse = requestBuilder.execute().actionGet(); + SearchSourceBuilder searchSource = new SearchSourceBuilder(); + searchSource.query(queryStringBuilder); + searchSource.sort("rowcount", SortOrder.DESC); + searchSource.from(i * 10); + searchSource.size(10); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("pages"); + searchRequest.types("row"); + searchRequest.source(searchSource); + SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); assertTrue(searchResponse.getHits().getTotalHits() > 0); } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java index 5948daa..1ee6bfb 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java @@ -1,6 +1,6 @@ package org.xbib.elx.common.test; -import static org.junit.Assert.assertEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.elasticsearch.action.admin.indices.create.CreateIndexAction; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -17,43 +17,51 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; -public class SimpleTest extends TestBase { +@ExtendWith(TestExtension.class) +class SimpleTest { + + private final TestExtension.Helper helper; + + SimpleTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void test() throws Exception { + void test() throws Exception { try { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices("test"); - client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); + helper.client("1").execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); } catch (IndexNotFoundException e) { // ignore if index not found } Settings indexSettings = Settings.settingsBuilder() - .put(super.getNodeSettings()) + .put(helper.getNodeSettings()) .put("index.analysis.analyzer.default.filter.0", "lowercase") .put("index.analysis.analyzer.default.filter.1", "trim") .put("index.analysis.analyzer.default.tokenizer", "keyword") .build(); CreateIndexRequest createIndexRequest = new CreateIndexRequest(); createIndexRequest.index("test").settings(indexSettings); - client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + helper.client("1").execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); IndexRequest indexRequest = new IndexRequest(); indexRequest.index("test").type("test").id("1") .source(XContentFactory.jsonBuilder().startObject().field("field", "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8").endObject()); - client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); + helper.client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); RefreshRequest refreshRequest = new RefreshRequest(); refreshRequest.indices("test"); - client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); + helper.client("1").execute(RefreshAction.INSTANCE, refreshRequest).actionGet(); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchQuery("field", "1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8")); SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("test").types("test"); searchRequest.source(builder); - String doc = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet() + String doc = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet() .getHits().getAt(0).getSourceAsString(); assertEquals(doc, "{\"field\":\"1%2fPJJP3JV2C24iDfEu9XpHBaYxXh%2fdHTbmchB35SDznXO2g8Vz4D7GTIvY54iMiX_149c95f02a8\"}"); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java deleted file mode 100644 index 12dc194..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/TestBase.java +++ /dev/null @@ -1,212 +0,0 @@ -package org.xbib.elx.common.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; - -public class TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - private static final Random random = new Random(); - - private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String cluster; - - private String host; - - private int port; - - @Before - public void startNodes() { - try { - logger.info("starting"); - setClusterName("test-cluster-" + System.getProperty("user.name")); - startNode("1"); - findNodeAddress(); - try { - ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) - .timeout(TimeValue.timeValueSeconds(30))).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); - } - ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); - ClusterStateResponse clusterStateResponse = - client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); - logger.info("host = {} port = {}", host, port); - - } catch (Throwable t) { - logger.error("startNodes failed", t); - } - } - - @After - public void stopNodes() { - try { - closeNodes(); - } catch (Exception e) { - logger.error("can not close nodes", e); - } finally { - try { - deleteFiles(); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected void setClusterName(String cluster) { - this.cluster = cluster; - } - - protected String getClusterName() { - return cluster; - } - - protected Settings getTransportSettings() { - return settingsBuilder() - .put("host", host) - .put("port", port) - .put("cluster.name", cluster) - .put("path.home", getHome()) - .build(); - } - - protected Settings getNodeSettings() { - return settingsBuilder() - .put("cluster.name", cluster) - //.put("cluster.routing.schedule", "50ms") - //.put("cluster.routing.allocation.disk.threshold_enabled", false) - //.put("discovery.zen.multicast.enabled", true) - //.put("discovery.zen.multicast.ping_timeout", "5s") - //.put("http.enabled", true) - //.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) - //.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low - //.put("index.number_of_replicas", 0) - .put("path.home", getHome()) - .build(); - } - - protected static String getHome() { - return System.getProperty("path.home", System.getProperty("user.dir")); - } - - protected void startNode(String id) { - buildNode(id).start(); - } - - protected AbstractClient client(String id) { - return clients.get(id); - } - - protected void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - Object obj = response.iterator().next().getTransport().getAddress() - .publishAddress(); - if (obj instanceof InetSocketTransportAddress) { - InetSocketTransportAddress address = (InetSocketTransportAddress) obj; - host = address.address().getHostName(); - port = address.address().getPort(); - } - } - - private Node buildNode(String id) { - Settings nodeSettings = settingsBuilder() - .put(getNodeSettings()) - .put("name", id) - .build(); - Node node = new MockNode(nodeSettings); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - logger.info("clients={}", clients); - return node; - } - - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - private void closeNodes() { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } -} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java new file mode 100644 index 0000000..fdf65e6 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java @@ -0,0 +1,216 @@ +package org.xbib.elx.common.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class TestExtension implements ParameterResolver, BeforeAllCallback, AfterAllCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private String home; + + private String cluster; + + private String host; + + private int port; + + private static final String key = "es-instance"; + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); + setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + setClusterName("test-cluster-" + System.getProperty("user.name")); + logger.info("starting cluster"); + deleteFiles(Paths.get(getHome() + "/data")); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + closeNodes(); + deleteFiles(Paths.get(getHome() + "/data")); + } + + private void setClusterName(String cluster) { + this.cluster = cluster; + } + + private String getClusterName() { + return cluster; + } + + private void setHome(String home) { + this.home = home; + } + + private String getHome() { + return home; + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + return new Helper(); + } + + class Helper { + + Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } + + void startNode(String id) { + buildNode(id).start(); + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java index 1bb681c..4f298aa 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java @@ -6,53 +6,51 @@ import org.elasticsearch.action.index.IndexAction; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import java.io.IOException; -public class WildcardTest extends TestBase { +@ExtendWith(TestExtension.class) +class WildcardTest { - /*protected Settings getNodeSettings() { - return Settings.settingsBuilder() - .put(super.getNodeSettings()) - .put("cluster.routing.allocation.disk.threshold_enabled", false) - .put("discovery.zen.multicast.enabled", false) - .put("http.enabled", false) - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - }*/ + private final TestExtension.Helper helper; - @Test - public void testWildcard() throws Exception { - index(client("1"), "1", "010"); - index(client("1"), "2", "0*0"); - // exact - validateCount(client("1"), QueryBuilders.queryStringQuery("010").defaultField("field"), 1); - validateCount(client("1"), QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1); - // pattern - validateCount(client("1"), QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2? - validateCount(client("1"), QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2? - validateCount(client("1"), QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2? - validateCount(client("1"), QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0); - validateCount(client("1"), QueryBuilders.queryStringQuery("*10").defaultField("field"), 1); - validateCount(client("1"), QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1); - validateCount(client("1"), QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1? - validateCount(client("1"), QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1? + WildcardTest(TestExtension.Helper helper) { + this.helper = helper; } - private void index(Client client, String id, String fieldValue) throws IOException { + @Test + void testWildcard() throws Exception { + ElasticsearchClient client = helper.client("1"); + index(client, "1", "010"); + index(client, "2", "0*0"); + // exact + validateCount(client, QueryBuilders.queryStringQuery("010").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("0\\*0").defaultField("field"), 1); + // pattern + validateCount(client, QueryBuilders.queryStringQuery("0*0").defaultField("field"), 1); // 2? + validateCount(client, QueryBuilders.queryStringQuery("0?0").defaultField("field"), 1); // 2? + validateCount(client, QueryBuilders.queryStringQuery("0**0").defaultField("field"), 1); // 2? + validateCount(client, QueryBuilders.queryStringQuery("0??0").defaultField("field"), 0); + validateCount(client, QueryBuilders.queryStringQuery("*10").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("*1*").defaultField("field"), 1); + validateCount(client, QueryBuilders.queryStringQuery("*\\*0").defaultField("field"), 0); // 1? + validateCount(client, QueryBuilders.queryStringQuery("*\\**").defaultField("field"), 0); // 1? + } + + private void index(ElasticsearchClient client, String id, String fieldValue) throws IOException { client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id) .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())).actionGet(); client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); } - private long count(Client client, QueryBuilder queryBuilder) { + private long count(ElasticsearchClient client, QueryBuilder queryBuilder) { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(queryBuilder); SearchRequest searchRequest = new SearchRequest(); @@ -62,7 +60,7 @@ public class WildcardTest extends TestBase { return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); } - private void validateCount(Client client, QueryBuilder queryBuilder, long expectedHits) { + private void validateCount(ElasticsearchClient client, QueryBuilder queryBuilder, long expectedHits) { final long actualHits = count(client, queryBuilder); if (actualHits != expectedHits) { throw new RuntimeException("actualHits=" + actualHits + ", expectedHits=" + expectedHits); diff --git a/elx-http/build.gradle b/elx-http/build.gradle new file mode 100644 index 0000000..021d75d --- /dev/null +++ b/elx-http/build.gradle @@ -0,0 +1,4 @@ +dependencies { + compile project(':elx-common') + compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" +} \ No newline at end of file diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java new file mode 100644 index 0000000..1f51857 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java @@ -0,0 +1,127 @@ +package org.xbib.elx.http; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.netty.http.client.Client; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.ServiceLoader; + +/** + * Elasticsearch HTTP client. + */ +public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient { + + private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class); + + private Client nettyHttpClient; + + private final ClassLoader classLoader; + + @SuppressWarnings("rawtypes") + private final Map actionMap; + + private String url; + + public ExtendedHttpClient() { + this.classLoader = ExtendedHttpClient.class.getClassLoader(); + this.actionMap = new HashMap<>(); + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public ExtendedHttpClient init(Settings settings) throws IOException { + super.init(settings); + if (settings == null) { + return null; + } + this.url = settings.get("url"); + ServiceLoader httpActionServiceLoader = ServiceLoader.load(HttpAction.class, classLoader); + for (HttpAction httpAction : httpActionServiceLoader) { + httpAction.setSettings(settings); + actionMap.put(httpAction.getActionInstance(), httpAction); + } + this.nettyHttpClient = Client.builder().enableDebug().build(); + logger.info("extended HTTP client initialized with {} actions", actionMap.size()); + return this; + } + + public Client internalClient() { + return nettyHttpClient; + } + + @Override + public ElasticsearchClient getClient() { + return this; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return this; + } + + @Override + protected void closeClient() throws IOException { + nettyHttpClient.shutdownGracefully(); + } + + @Override + public > ActionFuture + execute(Action action, Request request) { + PlainActionFuture actionFuture = PlainActionFuture.newFuture(); + execute(action, request, actionFuture); + return actionFuture; + } + + @Override + public > void + execute(Action action, Request request, ActionListener listener) { + doExecute(action, request, listener); + } + + @Override + public > RequestBuilder + prepareExecute(Action action) { + return action.newRequestBuilder(this); + } + + @Override + public ThreadPool threadPool() { + throw new UnsupportedOperationException(); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private > + void doExecute(Action action, R request, ActionListener listener) { + HttpAction httpAction = actionMap.get(action); + if (httpAction == null) { + throw new IllegalStateException("failed to find http action [" + action + "] to execute"); + } + try { + HttpActionContext httpActionContext = new HttpActionContext(this, request, url); + if (logger.isDebugEnabled()) { + logger.debug("submitting request {} to URL {}", request, url); + } + httpAction.execute(httpActionContext, listener); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java new file mode 100644 index 0000000..e91c923 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.ExtendedClientProvider; + +public class ExtendedHttpClientProvider implements ExtendedClientProvider { + @Override + public ExtendedHttpClient getExtendedClient() { + return new ExtendedHttpClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java new file mode 100644 index 0000000..1b6399b --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java @@ -0,0 +1,169 @@ +package org.xbib.elx.http; + +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpMethod; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.xbib.elx.http.util.CheckedFunction; +import org.xbib.netty.http.client.Request; +import org.xbib.netty.http.client.RequestBuilder; +import org.xbib.netty.http.client.transport.Transport; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +/** + * Base class for HTTP actions. + * + * @param the request type + * @param the response type + */ +public abstract class HttpAction { + + private final Logger logger = LogManager.getLogger(getClass().getName()); + + private static final String APPLICATION_JSON = "application/json"; + + private Settings settings; + + void setSettings(Settings settings) { + this.settings = settings; + } + + public abstract GenericAction getActionInstance(); + + public final ActionFuture execute(HttpActionContext httpActionContext) throws IOException { + PlainActionFuture future = PlainActionFuture.newFuture(); + execute(httpActionContext, future); + return future; + } + + public final void execute(HttpActionContext httpActionContext, ActionListener listener) throws IOException { + try { + ActionRequestValidationException validationException = httpActionContext.getRequest().validate(); + if (validationException != null) { + listener.onFailure(validationException); + return; + } + RequestBuilder httpRequestBuilder = + createHttpRequest(httpActionContext.getUrl(), httpActionContext.getRequest()); + httpRequestBuilder.setUserAgent("elx-http/1.0"); + Request httpRequest = httpRequestBuilder.build(); + if (logger.isTraceEnabled()) { + logger.trace("action = {} request = {}", this.getClass().getName(), httpRequest.toString()); + } + httpRequest.setResponseListener(fullHttpResponse -> { + try { + if (logger.isTraceEnabled()) { + logger.trace("got HTTP response: status code = " + fullHttpResponse.status().code() + + " headers = " + fullHttpResponse.headers().entries() + + " content = " + fullHttpResponse.content().toString(StandardCharsets.UTF_8)); + } + listener.onResponse(parseToResponse(httpActionContext.setHttpResponse(fullHttpResponse))); + } catch (Exception e) { + listener.onFailure(e); + } + }); + Transport transport = httpActionContext.getExtendedHttpClient().internalClient().execute(httpRequest); + httpActionContext.setHttpClientTransport(transport); + if (transport.isFailed()) { + listener.onFailure(transport.getFailure()); + } + } catch (Throwable e) { + if (listener != null) { + listener.onFailure(e); + } + throw new IOException(e); + } + } + + protected RequestBuilder newGetRequest(String url, String path) { + return Request.builder(HttpMethod.GET).url(url).uri(path); + } + + protected RequestBuilder newGetRequest(String url, String path, BytesReference content) { + return newRequest(HttpMethod.GET, url, path, content); + } + + protected RequestBuilder newHeadRequest(String url, String path) { + return newRequest(HttpMethod.HEAD, url, path); + } + + protected RequestBuilder newPostRequest(String url, String path) { + return newRequest(HttpMethod.POST, url, path); + } + + protected RequestBuilder newPostRequest(String url, String path, BytesReference content) { + return newRequest(HttpMethod.POST, url, path, content); + } + + protected RequestBuilder newPostRequest(String url, String path, String content) { + return newRequest(HttpMethod.POST, url, path, content); + } + + protected RequestBuilder newPutRequest(String url, String path) { + return newRequest(HttpMethod.PUT, url, path); + } + + protected RequestBuilder newPutRequest(String url, String path, String content) { + return newRequest(HttpMethod.PUT, url, path, content); + } + + protected RequestBuilder newPutRequest(String url, String path, BytesReference content) { + return newRequest(HttpMethod.PUT, url, path, content); + } + + protected RequestBuilder newDeleteRequest(String url, String path, BytesReference content) { + return newRequest(HttpMethod.DELETE, url, path, content); + } + + protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path) { + return Request.builder(method).url(baseUrl).uri(path); + } + + protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path, BytesReference content) { + return Request.builder(method).url(baseUrl).uri(path).content(content.toBytesRef().bytes, APPLICATION_JSON); + } + + protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path, String content) { + return Request.builder(method).url(baseUrl).uri(path).content(content, APPLICATION_JSON); + } + + protected RequestBuilder newRequest(HttpMethod method, String baseUrl, String path, ByteBuf byteBuf) { + return Request.builder(method).url(baseUrl).uri(path).content(byteBuf, APPLICATION_JSON); + } + + protected T parseToResponse(HttpActionContext httpActionContext) throws IOException { + String mediaType = httpActionContext.getHttpResponse().headers().get(HttpHeaderNames.CONTENT_TYPE); + // strip off "; charset=UTF-8" + int pos = mediaType.indexOf(";"); + mediaType = pos >= 0 ? mediaType.substring(0, pos) : mediaType; + XContentType xContentType = XContentType.fromRestContentType(mediaType); + if (xContentType == null) { + throw new IllegalStateException("unsupported content-type: " + mediaType); + } + String body = httpActionContext.getHttpResponse().content().toString(StandardCharsets.UTF_8); + T t; + try (XContentParser parser = xContentType.xContent().createParser(body)) { + t = entityParser().apply(parser); + } + return t; + } + + protected abstract RequestBuilder createHttpRequest(String baseUrl, R request) throws IOException; + + protected abstract CheckedFunction entityParser(); + +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java new file mode 100644 index 0000000..0a0abeb --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java @@ -0,0 +1,60 @@ +package org.xbib.elx.http; + +import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.xbib.netty.http.client.transport.Transport; + +/** + * HTTP action context. + * + * @param request type + * @param response type + */ +public class HttpActionContext { + + private final ExtendedHttpClient extendedHttpClient; + + private final R request; + + private final String url; + + private Transport httpClientTransport; + + private FullHttpResponse httpResponse; + + HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) { + this.extendedHttpClient = extendedHttpClient; + this.request = request; + this.url = url; + } + + public ExtendedHttpClient getExtendedHttpClient() { + return extendedHttpClient; + } + + public R getRequest() { + return request; + } + + public String getUrl() { + return url; + } + + public void setHttpClientTransport(Transport httpClientTransport) { + this.httpClientTransport = httpClientTransport; + } + + public Transport getHttpClientTransport() { + return httpClientTransport; + } + + public HttpActionContext setHttpResponse(FullHttpResponse fullHttpResponse) { + this.httpResponse = fullHttpResponse; + return this; + } + + public FullHttpResponse getHttpResponse() { + return httpResponse; + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java new file mode 100644 index 0000000..dbb9e53 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpGetAction.java @@ -0,0 +1,179 @@ +package org.xbib.elx.http.action.get; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.get.GetField; +import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; +import org.xbib.elx.http.util.CheckedFunction; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.RequestBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; + +public class HttpGetAction extends HttpAction { + + @Override + public GenericAction getActionInstance() { + return GetAction.INSTANCE; + } + + @Override + protected RequestBuilder createHttpRequest(String url, GetRequest request) { + return newGetRequest(url, request.index() + "/" + request.type() + "/" + request.id()); + } + + @Override + protected CheckedFunction entityParser() { + return this::fromXContent; + } + + public GetResponse fromXContent(XContentParser parser) throws IOException { + GetResult getResult = Helper.fromXContent(parser); + if (getResult.getIndex() == null && getResult.getType() == null && getResult.getId() == null) { + throw new ElasticsearchException(parser.getTokenLocation() + ":" + + String.format(Locale.ROOT, "Missing required fields [%s,%s,%s]", "_index", "_type", "_id")); + } + return new GetResponse(getResult); + } + + static class Helper { + + private static final Logger logger = LogManager.getLogger("helper"); + + static final String _INDEX = "_index"; + static final String _TYPE = "_type"; + static final String _ID = "_id"; + private static final String _VERSION = "_version"; + private static final String FOUND = "found"; + private static final String FIELDS = "fields"; + + static void ensureExpectedToken(XContentParser.Token expected, XContentParser.Token actual, Supplier location) { + if (actual != expected) { + String message = "Failed to parse object: expecting token of type [%s] but found [%s]"; + throw new ElasticsearchException(location.get() + ":" + String.format(Locale.ROOT, message, expected, actual)); + } + } + + static GetResult fromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, token, parser::getTokenLocation); + return fromXContentEmbedded(parser); + } + + static GetResult fromXContentEmbedded(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + return fromXContentEmbedded(parser, null, null, null); + } + + static GetResult fromXContentEmbedded(XContentParser parser, String index, String type, String id) throws IOException { + XContentParser.Token token = parser.currentToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + String currentFieldName = parser.currentName(); + long version = -1; + Boolean found = null; + BytesReference source = null; + Map fields = new HashMap<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (_INDEX.equals(currentFieldName)) { + index = parser.text(); + } else if (_TYPE.equals(currentFieldName)) { + type = parser.text(); + } else if (_ID.equals(currentFieldName)) { + id = parser.text(); + } else if (_VERSION.equals(currentFieldName)) { + version = parser.longValue(); + } else if (FOUND.equals(currentFieldName)) { + found = parser.booleanValue(); + } else { + fields.put(currentFieldName, new GetField(currentFieldName, Collections.singletonList(parser.objectText()))); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (SourceFieldMapper.NAME.equals(currentFieldName)) { + try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) { + builder.copyCurrentStructure(parser); + source = builder.bytes(); + } + } else if (FIELDS.equals(currentFieldName)) { + while(parser.nextToken() != XContentParser.Token.END_OBJECT) { + GetField getField = getFieldFromXContent(parser); + fields.put(getField.getName(), getField); + } + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("_ignored".equals(currentFieldName)) { + fields.put(currentFieldName, new GetField(currentFieldName, parser.list())); + } else { + parser.skipChildren(); + } + } + } + return new GetResult(index, type, id, version, found, source, fields); + } + + static GetField getFieldFromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + String fieldName = parser.currentName(); + XContentParser.Token token = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation); + List values = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + values.add(parseFieldsValue(parser)); + } + return new GetField(fieldName, values); + } + + static Object parseFieldsValue(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + Object value = null; + if (token == XContentParser.Token.VALUE_STRING) { + //binary values will be parsed back and returned as base64 strings when reading from json and yaml + value = parser.text(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.numberValue(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + value = parser.booleanValue(); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + //binary values will be parsed back and returned as BytesArray when reading from cbor and smile + value = new BytesArray(parser.binaryValue()); + } else if (token == XContentParser.Token.VALUE_NULL) { + value = null; + } else if (token == XContentParser.Token.START_OBJECT) { + value = parser.mapOrdered(); + } else if (token == XContentParser.Token.START_ARRAY) { + value = parser.listOrderedMap(); + } else { + throwUnknownToken(token, parser.getTokenLocation()); + } + return value; + } + + static void throwUnknownToken(XContentParser.Token token, XContentLocation location) { + String message = "Failed to parse object: unexpected token [%s] found"; + throw new ElasticsearchException(location + ":" + String.format(Locale.ROOT, message, token)); + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java new file mode 100644 index 0000000..3e1c6d1 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/get/HttpMultiGetAction.java @@ -0,0 +1,255 @@ +package org.xbib.elx.http.action.get; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.GenericAction; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetAction; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.get.GetField; +import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; +import org.xbib.elx.http.HttpAction; +import org.xbib.elx.http.action.search.HttpSearchAction; +import org.xbib.elx.http.util.CheckedFunction; +import org.xbib.netty.http.client.RequestBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; + +public class HttpMultiGetAction extends HttpAction { + + @Override + public GenericAction getActionInstance() { + return MultiGetAction.INSTANCE; + } + + @Override + protected RequestBuilder createHttpRequest(String url, MultiGetRequest request) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startArray("docs"); + for (MultiGetRequest.Item item : request.getItems()) { + builder.startObject() + .field("_index", item.index()) + .field("_type", item.type()) + .field("_id", item.id()); + if (item.fields() != null) { + builder.array("fields", item.fields()); + } + builder.endObject(); + } + builder.endArray().endObject(); + return newPostRequest(url, "_mget", builder.bytes()); + } + + @Override + protected CheckedFunction entityParser() { + return Helper::fromXContent; + } + + static class Helper { + + private static final ParseField INDEX = new ParseField("_index"); + private static final ParseField TYPE = new ParseField("_type"); + private static final ParseField ID = new ParseField("_id"); + private static final ParseField ERROR = new ParseField("error"); + private static final ParseField DOCS = new ParseField("docs"); + + static final String _INDEX = "_index"; + static final String _TYPE = "_type"; + static final String _ID = "_id"; + private static final String _VERSION = "_version"; + private static final String FOUND = "found"; + private static final String FIELDS = "fields"; + + + static MultiGetResponse fromXContent(XContentParser parser) throws IOException { + String currentFieldName = null; + List items = new ArrayList<>(); + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + switch (token) { + case FIELD_NAME: + currentFieldName = parser.currentName(); + break; + case START_ARRAY: + if (DOCS.getPreferredName().equals(currentFieldName)) { + for (token = parser.nextToken(); token != XContentParser.Token.END_ARRAY; token = parser.nextToken()) { + if (token == XContentParser.Token.START_OBJECT) { + items.add(parseItem(parser)); + } + } + } + break; + default: + break; + } + } + return new MultiGetResponse(items.toArray(new MultiGetItemResponse[0])); + } + + private static MultiGetItemResponse parseItem(XContentParser parser) throws IOException { + String currentFieldName = null; + String index = null; + String type = null; + String id = null; + ElasticsearchException exception = null; + GetResult getResult = null; + ParseFieldMatcher matcher = new ParseFieldMatcher(Settings.EMPTY); + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + switch (token) { + case FIELD_NAME: + currentFieldName = parser.currentName(); + getResult = fromXContentEmbedded(parser, index, type, id); + break; + case VALUE_STRING: + if (matcher.match(currentFieldName, INDEX)) { + index = parser.text(); + } else if (matcher.match(currentFieldName, TYPE)) { + type = parser.text(); + } else if (matcher.match(currentFieldName, ID)) { + id = parser.text(); + } + break; + case START_OBJECT: + if (matcher.match(currentFieldName, ERROR)) { + exception = HttpSearchAction.Helper.elasticsearchExceptionFromXContent(parser); + } + break; + default: + // If unknown tokens are encounter then these should be ignored, because + // this is parsing logic on the client side. + break; + } + if (getResult != null) { + break; + } + } + if (exception != null) { + return new MultiGetItemResponse(null, new MultiGetResponse.Failure(index, type, id, exception)); + } else { + GetResponse getResponse = new GetResponse(getResult); + return new MultiGetItemResponse(getResponse, null); + } + } + + static void ensureExpectedToken(XContentParser.Token expected, XContentParser.Token actual, Supplier location) { + if (actual != expected) { + String message = "Failed to parse object: expecting token of type [%s] but found [%s]"; + throw new ElasticsearchException(location.get() + ":" + String.format(Locale.ROOT, message, expected, actual)); + } + } + + static GetResult fromXContentEmbedded(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + return fromXContentEmbedded(parser, null, null, null); + } + + static GetResult fromXContentEmbedded(XContentParser parser, String index, String type, String id) throws IOException { + XContentParser.Token token = parser.currentToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + String currentFieldName = parser.currentName(); + long version = -1; + Boolean found = null; + BytesReference source = null; + Map fields = new HashMap<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (_INDEX.equals(currentFieldName)) { + index = parser.text(); + } else if (_TYPE.equals(currentFieldName)) { + type = parser.text(); + } else if (_ID.equals(currentFieldName)) { + id = parser.text(); + } else if (_VERSION.equals(currentFieldName)) { + version = parser.longValue(); + } else if (FOUND.equals(currentFieldName)) { + found = parser.booleanValue(); + } else { + fields.put(currentFieldName, new GetField(currentFieldName, Collections.singletonList(parser.objectText()))); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (SourceFieldMapper.NAME.equals(currentFieldName)) { + try (XContentBuilder builder = XContentBuilder.builder(parser.contentType().xContent())) { + builder.copyCurrentStructure(parser); + source = builder.bytes(); + } + } else if (FIELDS.equals(currentFieldName)) { + while(parser.nextToken() != XContentParser.Token.END_OBJECT) { + GetField getField = getFieldFromXContent(parser); + fields.put(getField.getName(), getField); + } + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if ("_ignored".equals(currentFieldName)) { + fields.put(currentFieldName, new GetField(currentFieldName, parser.list())); + } else { + parser.skipChildren(); + } + } + } + return new GetResult(index, type, id, version, found, source, fields); + } + + static GetField getFieldFromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + String fieldName = parser.currentName(); + XContentParser.Token token = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation); + List values = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + values.add(parseFieldsValue(parser)); + } + return new GetField(fieldName, values); + } + + static Object parseFieldsValue(XContentParser parser) throws IOException { + XContentParser.Token token = parser.currentToken(); + Object value = null; + if (token == XContentParser.Token.VALUE_STRING) { + //binary values will be parsed back and returned as base64 strings when reading from json and yaml + value = parser.text(); + } else if (token == XContentParser.Token.VALUE_NUMBER) { + value = parser.numberValue(); + } else if (token == XContentParser.Token.VALUE_BOOLEAN) { + value = parser.booleanValue(); + } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { + //binary values will be parsed back and returned as BytesArray when reading from cbor and smile + value = new BytesArray(parser.binaryValue()); + } else if (token == XContentParser.Token.VALUE_NULL) { + value = null; + } else if (token == XContentParser.Token.START_OBJECT) { + value = parser.mapOrdered(); + } else if (token == XContentParser.Token.START_ARRAY) { + value = parser.listOrderedMap(); + } else { + throwUnknownToken(token, parser.getTokenLocation()); + } + return value; + } + + static void throwUnknownToken(XContentParser.Token token, XContentLocation location) { + String message = "Failed to parse object: unexpected token [%s] found"; + throw new ElasticsearchException(location + ":" + String.format(Locale.ROOT, message, token)); + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java new file mode 100644 index 0000000..58a5040 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java @@ -0,0 +1,597 @@ +package org.xbib.elx.http.action.search; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.internal.SourceFieldMapper; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.internal.InternalSearchHit; +import org.elasticsearch.search.internal.InternalSearchHits; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry; +import org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option; +import org.xbib.elx.http.util.CheckedFunction; +import org.xbib.elx.http.HttpAction; +import org.xbib.elx.http.util.ObjectParser; +import org.xbib.elx.http.util.XContentParserUtils; +import org.xbib.netty.http.client.RequestBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.xbib.elx.http.util.ObjectParser.ValueType.STRING; +import static org.xbib.elx.http.util.XContentParserUtils.ensureExpectedToken; + +public class HttpSearchAction extends HttpAction { + + @Override + public SearchAction getActionInstance() { + return SearchAction.INSTANCE; + } + + @Override + protected RequestBuilder createHttpRequest(String url, SearchRequest request) { + String index = request.indices() != null ? "/" + String.join(",", request.indices()) : ""; + return newPostRequest(url, index + "/_search", request.source()); + } + + @Override + protected CheckedFunction entityParser() { + return Helper::fromXContent; + } + + public static class Helper { + + private static final Logger logger = LogManager.getLogger("helper"); + + private static final ParseField SCROLL_ID = new ParseField("_scroll_id"); + private static final ParseField TOOK = new ParseField("took"); + private static final ParseField TIMED_OUT = new ParseField("timed_out"); + private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early"); + + private static final ParseField _SHARDS_FIELD = new ParseField("_shards"); + private static final ParseField TOTAL_FIELD = new ParseField("total"); + private static final ParseField SUCCESSFUL_FIELD = new ParseField("successful"); + private static final ParseField SKIPPED_FIELD = new ParseField("skipped"); + private static final ParseField FAILED_FIELD = new ParseField("failed"); + private static final ParseField FAILURES_FIELD = new ParseField("failures"); + + private static final String HITS = "hits"; + + private static final String TOTAL = "total"; + private static final String MAX_SCORE = "max_score"; + + private static final String _NESTED = "_nested"; + + private static final String _INDEX = "_index"; + private static final String _TYPE = "_type"; + private static final String _ID = "_id"; + private static final String _VERSION = "_version"; + private static final String _SCORE = "_score"; + private static final String FIELDS = "fields"; + private static final String HIGHLIGHT = "highlight"; + private static final String SORT = "sort"; + private static final String MATCHED_QUERIES = "matched_queries"; + private static final String _EXPLANATION = "_explanation"; + private static final String INNER_HITS = "inner_hits"; + private static final String _SHARD = "_shard"; + private static final String _NODE = "_node"; + + private static final String AGGREGATIONS_FIELD = "aggregations"; + + private static final String TYPED_KEYS_DELIMITER = "#"; + + private static final String SUGGEST_NAME = "suggest"; + + private static final String REASON_FIELD = "reason"; + private static final String NODE_FIELD = "node"; + private static final String INDEX_FIELD = "index"; + private static final String SHARD_FIELD = "shard"; + + private static final String TYPE = "type"; + private static final String REASON = "reason"; + private static final String CAUSED_BY = "caused_by"; + private static final String STACK_TRACE = "stack_trace"; + private static final String HEADER = "header"; + private static final String ROOT_CAUSE = "root_cause"; + + private static ObjectParser, Void> MAP_PARSER = + new ObjectParser<>("innerHitParser", true, HashMap::new); + + + static { + declareInnerHitsParseFields(MAP_PARSER); + } + + public static SearchResponse fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + parser.nextToken(); + return innerFromXContent(parser); + } + + static SearchResponse innerFromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + String currentFieldName = parser.currentName(); + InternalSearchHits hits = null; + InternalAggregations aggs = null; + Suggest suggest = null; + boolean timedOut = false; + Boolean terminatedEarly = null; + long tookInMillis = -1; + int successfulShards = -1; + int totalShards = -1; + String scrollId = null; + List failures = new ArrayList<>(); + ParseFieldMatcher matcher = new ParseFieldMatcher(Settings.EMPTY); + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (matcher.match(currentFieldName, SCROLL_ID)) { + scrollId = parser.text(); + } else if (matcher.match(currentFieldName, TOOK)) { + tookInMillis = parser.longValue(); + } else if (matcher.match(currentFieldName, TIMED_OUT)) { + timedOut = parser.booleanValue(); + } else if (matcher.match(currentFieldName, TERMINATED_EARLY)) { + terminatedEarly = parser.booleanValue(); + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (HITS.equals(currentFieldName)) { + logger.debug("searchHitsFromXContent"); + hits = searchHitsFromXContent(parser); + } else if (AGGREGATIONS_FIELD.equals(currentFieldName)) { + aggs = aggregationsFromXContent(parser); + } else if (SUGGEST_NAME.equals(currentFieldName)) { + suggest = suggestFromXContent(parser); + } else if (matcher.match(currentFieldName, _SHARDS_FIELD)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (matcher.match(currentFieldName, FAILED_FIELD)) { + parser.intValue(); // we don't need it but need to consume it + } else if (matcher.match(currentFieldName, SUCCESSFUL_FIELD)) { + successfulShards = parser.intValue(); + } else if (matcher.match(currentFieldName, TOTAL_FIELD)) { + totalShards = parser.intValue(); + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (matcher.match(currentFieldName, FAILURES_FIELD)) { + while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + failures.add(shardSearchFailureFromXContent(parser)); + } + } else { + parser.skipChildren(); + } + } else { + parser.skipChildren(); + } + } + } else { + parser.skipChildren(); + } + } + } + // TODO profileResults + InternalSearchResponse internalResponse = new InternalSearchResponse(hits, aggs, suggest, + null, timedOut, terminatedEarly); + return new SearchResponse(internalResponse, scrollId, totalShards, successfulShards, tookInMillis, + failures.toArray(ShardSearchFailure.EMPTY_ARRAY)); + } + + static InternalSearchHits searchHitsFromXContent(XContentParser parser) throws IOException { + if (parser.currentToken() != XContentParser.Token.START_OBJECT) { + parser.nextToken(); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + } + XContentParser.Token token = parser.currentToken(); + String currentFieldName = null; + List hits = new ArrayList<>(); + long totalHits = -1L; + float maxScore = 0f; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (TOTAL.equals(currentFieldName)) { + totalHits = parser.longValue(); + } else if (MAX_SCORE.equals(currentFieldName)) { + maxScore = parser.floatValue(); + } + } else if (token == XContentParser.Token.VALUE_NULL) { + if (MAX_SCORE.equals(currentFieldName)) { + maxScore = Float.NaN; // NaN gets rendered as null-field + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (HITS.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + logger.debug("searchHitFromXContent"); + hits.add(searchHitFromXContent(parser)); + } + } else { + parser.skipChildren(); + } + } + } + InternalSearchHit[] internalSearchHits = hits.toArray(new InternalSearchHit[0]); + return new InternalSearchHits(internalSearchHits, totalHits, maxScore); + } + + static InternalSearchHit searchHitFromXContent(XContentParser parser) { + return createFromMap(MAP_PARSER.apply(parser, null)); + } + + static InternalSearchHit createFromMap(Map values) { + logger.debug("values = {}", values); + String id = get(_ID, values, null); + Text type = get(_TYPE, values, null); + InternalSearchHit.InternalNestedIdentity nestedIdentity = get(_NESTED, values, null); + Map fields = get(FIELDS, values, Collections.emptyMap()); + InternalSearchHit searchHit = new InternalSearchHit(-1, id, type, nestedIdentity, fields); + String index = get(_INDEX, values, null); + ShardId shardId = get(_SHARD, values, null); + String nodeId = get(_NODE, values, null); + if (shardId != null && nodeId != null) { + assert shardId.index().getName().equals(index); + searchHit.shard(new SearchShardTarget(nodeId, index, shardId.id())); + } + searchHit.score(get(_SCORE, values, Float.NaN)); + searchHit.version(get(_VERSION, values, -1L)); + searchHit.sortValues(get(SORT, values, new Object[0])); + searchHit.highlightFields(get(HIGHLIGHT, values, null)); + searchHit.sourceRef(get(SourceFieldMapper.NAME, values, null)); + searchHit.explanation(get(_EXPLANATION, values, null)); + searchHit.setInnerHits(get(INNER_HITS, values, null)); + List matchedQueries = get(MATCHED_QUERIES, values, null); + if (matchedQueries != null) { + searchHit.matchedQueries(matchedQueries.toArray(new String[0])); + } + return searchHit; + } + + @SuppressWarnings("unchecked") + private static T get(String key, Map map, T defaultValue) { + return (T) map.getOrDefault(key, defaultValue); + } + + static InternalAggregations aggregationsFromXContent(XContentParser parser) throws IOException { + final List aggregations = new ArrayList<>(); + XContentParser.Token token; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.START_OBJECT) { + SetOnce typedAgg = new SetOnce<>(); + String currentField = parser.currentName(); + XContentParserUtils.parseTypedKeysObject(parser, TYPED_KEYS_DELIMITER, InternalAggregation.class, typedAgg::set); + if (typedAgg.get() != null) { + aggregations.add(typedAgg.get()); + } else { + throw new ElasticsearchException(parser.getTokenLocation() + ":" + + String.format(Locale.ROOT, "Could not parse aggregation keyed as [%s]", currentField)); + } + } + } + return new InternalAggregations(aggregations); + } + + static Suggest suggestFromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + List>> suggestions = new ArrayList<>(); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + String currentField = parser.currentName(); + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.nextToken(), parser::getTokenLocation); + Suggest.Suggestion> suggestion = suggestionFromXContent(parser); + if (suggestion != null) { + suggestions.add(suggestion); + } else { + throw new ElasticsearchException(parser.getTokenLocation() + ":" + + String.format(Locale.ROOT, "Could not parse suggestion keyed as [%s]", currentField)); + } + } + return new Suggest(suggestions); + } + + static Suggest.Suggestion> suggestionFromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser::getTokenLocation); + SetOnce suggestion = new SetOnce<>(); + XContentParserUtils.parseTypedKeysObject(parser, "#", Suggest.Suggestion.class, suggestion::set); + return suggestion.get(); + } + + static ShardSearchFailure shardSearchFailureFromXContent(XContentParser parser) throws IOException { + XContentParser.Token token; + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + String currentFieldName = null; + int shardId = -1; + String indexName = null; + String nodeId = null; + ElasticsearchException exception = null; + while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (SHARD_FIELD.equals(currentFieldName)) { + shardId = parser.intValue(); + } else if (INDEX_FIELD.equals(currentFieldName)) { + indexName = parser.text(); + } else if (NODE_FIELD.equals(currentFieldName)) { + nodeId = parser.text(); + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (REASON_FIELD.equals(currentFieldName)) { + exception = elasticsearchExceptionFromXContent(parser); + } else { + parser.skipChildren(); + } + } else { + parser.skipChildren(); + } + } + SearchShardTarget searchShardTarget = null; + if (nodeId != null) { + searchShardTarget = new SearchShardTarget(nodeId, indexName, shardId); + } + return new ShardSearchFailure(exception, searchShardTarget); + } + + public static ElasticsearchException elasticsearchExceptionFromXContent(XContentParser parser) throws IOException { + XContentParser.Token token = parser.nextToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + return elasticsearchExceptionFromXContent(parser, false); + } + + static ElasticsearchException elasticsearchExceptionFromXContent(XContentParser parser, boolean parseRootCauses) + throws IOException { + XContentParser.Token token = parser.currentToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + + String type = null, reason = null, stack = null; + ElasticsearchException cause = null; + Map> metadata = new HashMap<>(); + Map> headers = new HashMap<>(); + List rootCauses = new ArrayList<>(); + + for (; token == XContentParser.Token.FIELD_NAME; token = parser.nextToken()) { + String currentFieldName = parser.currentName(); + token = parser.nextToken(); + + if (token.isValue()) { + if (TYPE.equals(currentFieldName)) { + type = parser.text(); + } else if (REASON.equals(currentFieldName)) { + reason = parser.text(); + } else if (STACK_TRACE.equals(currentFieldName)) { + stack = parser.text(); + } else if (token == XContentParser.Token.VALUE_STRING) { + metadata.put(currentFieldName, Collections.singletonList(parser.text())); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (CAUSED_BY.equals(currentFieldName)) { + cause = elasticsearchExceptionFromXContent(parser); + } else if (HEADER.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + List values = headers.getOrDefault(currentFieldName, new ArrayList<>()); + if (token == XContentParser.Token.VALUE_STRING) { + values.add(parser.text()); + } else if (token == XContentParser.Token.START_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + values.add(parser.text()); + } else { + parser.skipChildren(); + } + } + } else if (token == XContentParser.Token.START_OBJECT) { + parser.skipChildren(); + } + headers.put(currentFieldName, values); + } + } + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (parseRootCauses && ROOT_CAUSE.equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + rootCauses.add(elasticsearchExceptionFromXContent(parser)); + } + } else { + List values = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token == XContentParser.Token.VALUE_STRING) { + values.add(parser.text()); + } else { + parser.skipChildren(); + } + } + if (values.size() > 0) { + if (metadata.containsKey(currentFieldName)) { + values.addAll(metadata.get(currentFieldName)); + } + metadata.put(currentFieldName, values); + } + } + } + } + ElasticsearchException e = new ElasticsearchException(buildMessage(type, reason, stack), cause); + for (Map.Entry> header : headers.entrySet()) { + e.addHeader(header.getKey(), header.getValue()); + } + for (ElasticsearchException rootCause : rootCauses) { + e.addSuppressed(rootCause); + } + return e; + } + + static String buildMessage(String type, String reason, String stack) { + StringBuilder message = new StringBuilder("Elasticsearch exception ["); + message.append(TYPE).append('=').append(type).append(", "); + message.append(REASON).append('=').append(reason); + if (stack != null) { + message.append(", ").append(STACK_TRACE).append('=').append(stack); + } + message.append(']'); + return message.toString(); + } + + private static void declareInnerHitsParseFields(ObjectParser, Void> parser) { + declareMetaDataFields(parser); + parser.declareString((map, value) -> map.put(_TYPE, new Text(value)), new ParseField(_TYPE)); + parser.declareString((map, value) -> map.put(_INDEX, value), new ParseField(_INDEX)); + parser.declareString((map, value) -> map.put(_ID, value), new ParseField(_ID)); + parser.declareString((map, value) -> map.put(_NODE, value), new ParseField(_NODE)); + parser.declareField((map, value) -> map.put(_SCORE, value), SearchHit::parseScore, new ParseField(_SCORE), + ObjectParser.ValueType.FLOAT_OR_NULL); + parser.declareLong((map, value) -> map.put(_VERSION, value), new ParseField(_VERSION)); + parser.declareField((map, value) -> map.put(_SHARD, value), (p, c) -> ShardId.fromString(p.text()), + new ParseField(_SHARD), STRING); + parser.declareObject((map, value) -> map.put(SourceFieldMapper.NAME, value), (p, c) -> parseSourceBytes(p), + new ParseField(SourceFieldMapper.NAME)); + parser.declareObject((map, value) -> map.put(HIGHLIGHT, value), (p, c) -> parseHighlightFields(p), + new ParseField(HIGHLIGHT)); + parser.declareObject((map, value) -> { + Map fieldMap = get(FIELDS, map, new HashMap()); + fieldMap.putAll(value); + map.put(FIELDS, fieldMap); + }, (p, c) -> parseFields(p), new ParseField(FIELDS)); + parser.declareObject((map, value) -> map.put(_EXPLANATION, value), (p, c) -> parseExplanation(p), + new ParseField(_EXPLANATION)); + parser.declareObject((map, value) -> map.put(_NESTED, value), SearchHit.NestedIdentity::fromXContent, + new ParseField(_NESTED)); + parser.declareObject((map, value) -> map.put(INNER_HITS, value), (p,c) -> parseInnerHits(p), + new ParseField(INNER_HITS)); + parser.declareStringArray((map, list) -> map.put(MATCHED_QUERIES, list), new ParseField(MATCHED_QUERIES)); + parser.declareField((map, list) -> map.put(SORT, list), SearchSortValues::fromXContent, new ParseField(SORT), + ObjectParser.ValueType.OBJECT_ARRAY); + } + + private static void declareMetaDataFields(ObjectParser, Void> parser) { + for (String metadatafield : MapperService.getAllMetaFields()) { + if (!metadatafield.equals(_ID) && !metadatafield.equals(_INDEX) && !metadatafield.equals(_TYPE)) { + parser.declareField((map, field) -> { + @SuppressWarnings("unchecked") + Map fieldMap = (Map) map.computeIfAbsent(FIELDS, + v -> new HashMap()); + fieldMap.put(field.getName(), field); + }, (p, c) -> { + List values = new ArrayList<>(); + values.add(parseFieldsValue(p)); + return new InternalSearchHit(metadatafield, values); + }, new ParseField(metadatafield), ObjectParser.ValueType.VALUE); + } + } + } + + private static Map parseFields(XContentParser parser) throws IOException { + Map fields = new HashMap<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + SearchHitField field = SearchHitField.fromXContent(parser); + fields.put(field.getName(), field); + } + return fields; + } + + private static Map parseInnerHits(XContentParser parser) throws IOException { + Map innerHits = new HashMap<>(); + while ((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + String name = parser.currentName(); + ensureExpectedToken(Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + ensureFieldName(parser, parser.nextToken(), SearchHits.Fields.HITS); + innerHits.put(name, SearchHits.fromXContent(parser)); + ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation); + } + return innerHits; + } + + private static Map parseHighlightFields(XContentParser parser) throws IOException { + Map highlightFields = new HashMap<>(); + while((parser.nextToken()) != XContentParser.Token.END_OBJECT) { + HighlightField highlightField = HighlightField.fromXContent(parser); + highlightFields.put(highlightField.getName(), highlightField); + } + return highlightFields; + } + + private static Explanation parseExplanation(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + XContentParser.Token token; + Float value = null; + String description = null; + List details = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + String currentFieldName = parser.currentName(); + token = parser.nextToken(); + if (Fields.VALUE.equals(currentFieldName)) { + value = parser.floatValue(); + } else if (Fields.DESCRIPTION.equals(currentFieldName)) { + description = parser.textOrNull(); + } else if (Fields.DETAILS.equals(currentFieldName)) { + ensureExpectedToken(XContentParser.Token.START_ARRAY, token, parser::getTokenLocation); + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + details.add(parseExplanation(parser)); + } + } else { + parser.skipChildren(); + } + } + if (value == null) { + throw new ParsingException(parser.getTokenLocation(), "missing explanation value"); + } + if (description == null) { + throw new ParsingException(parser.getTokenLocation(), "missing explanation description"); + } + return Explanation.match(value, description, details); + } + + private void buildExplanation(XContentBuilder builder, Explanation explanation) throws IOException { + builder.startObject(); + builder.field(Fields.VALUE, explanation.getValue()); + builder.field(Fields.DESCRIPTION, explanation.getDescription()); + Explanation[] innerExps = explanation.getDetails(); + if (innerExps != null) { + builder.startArray(Fields.DETAILS); + for (Explanation exp : innerExps) { + buildExplanation(builder, exp); + } + builder.endArray(); + } + builder.endObject(); + } + + } +} + diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/AbstractObjectParser.java b/elx-http/src/main/java/org/xbib/elx/http/util/AbstractObjectParser.java new file mode 100644 index 0000000..1f854f1 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/AbstractObjectParser.java @@ -0,0 +1,217 @@ +package org.xbib.elx.http.util; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; + +public abstract class AbstractObjectParser + implements BiFunction, ContextParser { + + /** + * Declare some field. Usually it is easier to use {@link #declareString(BiConsumer, ParseField)} or + * {@link #declareObject(BiConsumer, ContextParser, ParseField)} rather than call this directly. + */ + public abstract void declareField(BiConsumer consumer, ContextParser parser, ParseField parseField, + ObjectParser.ValueType type); + + /** + * Declares named objects in the style of aggregations. These are named + * inside and object like this: + * + * 
+     * 
+     * {
+     *   "aggregations": {
+     *     "name_1": { "aggregation_type": {} },
+     *     "name_2": { "aggregation_type": {} },
+     *     "name_3": { "aggregation_type": {} }
+     *     }
+     *   }
+     * }
+     * 
+     *
+ * + * Unlike the other version of this method, "ordered" mode (arrays of + * objects) is not supported. + * + * See NamedObjectHolder in ObjectParserTests for examples of how to invoke + * this. + * + * @param consumer + * sets the values once they have been parsed + * @param namedObjectParser + * parses each named object + * @param parseField + * the field to parse + */ + public abstract void declareNamedObjects(BiConsumer> consumer, + ObjectParser.NamedObjectParser namedObjectParser, + ParseField parseField); + + /** + * Declares named objects in the style of highlighting's field element. + * These are usually named inside and object like this: + * + * 
+     * 
+     * {
+     *   "highlight": {
+     *     "fields": {        <------ this one
+     *       "title": {},
+     *       "body": {},
+     *       "category": {}
+     *     }
+     *   }
+     * }
+     * 
+     *
+ * + * but, when order is important, some may be written this way: + * + * 
+     * 
+     * {
+     *   "highlight": {
+     *     "fields": [        <------ this one
+     *       {"title": {}},
+     *       {"body": {}},
+     *       {"category": {}}
+     *     ]
+     *   }
+     * }
+     * 
+     *
+ * + * This is because json doesn't enforce ordering. Elasticsearch reads it in + * the order sent but tools that generate json are free to put object + * members in an unordered Map, jumbling them. Thus, if you care about order + * you can send the object in the second way. + * + * See NamedObjectHolder in ObjectParserTests for examples of how to invoke + * this. + * + * @param consumer + * sets the values once they have been parsed + * @param namedObjectParser + * parses each named object + * @param orderedModeCallback + * called when the named object is parsed using the "ordered" + * mode (the array of objects) + * @param parseField + * the field to parse + */ + public abstract void declareNamedObjects(BiConsumer> consumer, + ObjectParser.NamedObjectParser namedObjectParser, + Consumer orderedModeCallback, + ParseField parseField); + + public abstract String getName(); + + public void declareField(BiConsumer consumer, CheckedFunction parser, + ParseField parseField, ObjectParser.ValueType type) { + if (parser == null) { + throw new IllegalArgumentException("[parser] is required"); + } + declareField(consumer, (p, c) -> parser.apply(p), parseField, type); + } + + public void declareObject(BiConsumer consumer, ContextParser objectParser, ParseField field) { + declareField(consumer, (p, c) -> objectParser.parse(p, c), field, ObjectParser.ValueType.OBJECT); + } + + public void declareFloat(BiConsumer consumer, ParseField field) { + // Using a method reference here angers some compilers + declareField(consumer, p -> p.floatValue(), field, ObjectParser.ValueType.FLOAT); + } + + public void declareDouble(BiConsumer consumer, ParseField field) { + // Using a method reference here angers some compilers + declareField(consumer, p -> p.doubleValue(), field, ObjectParser.ValueType.DOUBLE); + } + + public void declareLong(BiConsumer consumer, ParseField field) { + // Using a method reference here angers some compilers + declareField(consumer, p -> p.longValue(), field, ObjectParser.ValueType.LONG); + } + + public void declareInt(BiConsumer consumer, ParseField field) { + // Using a method reference here angers some compilers + declareField(consumer, p -> p.intValue(), field, ObjectParser.ValueType.INT); + } + + public void declareString(BiConsumer consumer, ParseField field) { + declareField(consumer, XContentParser::text, field, ObjectParser.ValueType.STRING); + } + + public void declareStringOrNull(BiConsumer consumer, ParseField field) { + declareField(consumer, (p) -> p.currentToken() == XContentParser.Token.VALUE_NULL ? null : p.text(), field, + ObjectParser.ValueType.STRING_OR_NULL); + } + + public void declareBoolean(BiConsumer consumer, ParseField field) { + declareField(consumer, XContentParser::booleanValue, field, ObjectParser.ValueType.BOOLEAN); + } + + public void declareObjectArray(BiConsumer> consumer, ContextParser objectParser, + ParseField field) { + declareFieldArray(consumer, objectParser, field, ObjectParser.ValueType.OBJECT_ARRAY); + } + + public void declareStringArray(BiConsumer> consumer, ParseField field) { + declareFieldArray(consumer, (p, c) -> p.text(), field, ObjectParser.ValueType.STRING_ARRAY); + } + + public void declareDoubleArray(BiConsumer> consumer, ParseField field) { + declareFieldArray(consumer, (p, c) -> p.doubleValue(), field, ObjectParser.ValueType.DOUBLE_ARRAY); + } + + public void declareFloatArray(BiConsumer> consumer, ParseField field) { + declareFieldArray(consumer, (p, c) -> p.floatValue(), field, ObjectParser.ValueType.FLOAT_ARRAY); + } + + public void declareLongArray(BiConsumer> consumer, ParseField field) { + declareFieldArray(consumer, (p, c) -> p.longValue(), field, ObjectParser.ValueType.LONG_ARRAY); + } + + public void declareIntArray(BiConsumer> consumer, ParseField field) { + declareFieldArray(consumer, (p, c) -> p.intValue(), field, ObjectParser.ValueType.INT_ARRAY); + } + + /** + * Declares a field that can contain an array of elements listed in the type ValueType enum + */ + public void declareFieldArray(BiConsumer> consumer, ContextParser itemParser, + ParseField field, ObjectParser.ValueType type) { + declareField(consumer, (p, c) -> parseArray(p, () -> itemParser.parse(p, c)), field, type); + } + + private interface IOSupplier { + T get() throws IOException; + } + + private static List parseArray(XContentParser parser, IOSupplier supplier) throws IOException { + List list = new ArrayList<>(); + if (parser.currentToken().isValue() + || parser.currentToken() == XContentParser.Token.VALUE_NULL + || parser.currentToken() == XContentParser.Token.START_OBJECT) { + list.add(supplier.get()); // single value + } else { + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + if (parser.currentToken().isValue() + || parser.currentToken() == XContentParser.Token.VALUE_NULL + || parser.currentToken() == XContentParser.Token.START_OBJECT) { + list.add(supplier.get()); + } else { + throw new IllegalStateException("expected value but got [" + parser.currentToken() + "]"); + } + } + } + return list; + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/CheckedBiConsumer.java b/elx-http/src/main/java/org/xbib/elx/http/util/CheckedBiConsumer.java new file mode 100644 index 0000000..7213e9d --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/CheckedBiConsumer.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http.util; + +import java.util.function.BiConsumer; + +/** + * A {@link BiConsumer}-like interface which allows throwing checked exceptions. + */ +@FunctionalInterface +public interface CheckedBiConsumer { + void accept(T t, U u) throws E; +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/CheckedFunction.java b/elx-http/src/main/java/org/xbib/elx/http/util/CheckedFunction.java new file mode 100644 index 0000000..a2e4d8f --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/CheckedFunction.java @@ -0,0 +1,6 @@ +package org.xbib.elx.http.util; + +@FunctionalInterface +public interface CheckedFunction { + R apply(T t) throws E; +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/ContextParser.java b/elx-http/src/main/java/org/xbib/elx/http/util/ContextParser.java new file mode 100644 index 0000000..08534aa --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/ContextParser.java @@ -0,0 +1,13 @@ +package org.xbib.elx.http.util; + +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * Reads an object from a parser using some context. + */ +@FunctionalInterface +public interface ContextParser { + T parse(XContentParser p, Context c) throws IOException; +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/NamedObjectNotFoundException.java b/elx-http/src/main/java/org/xbib/elx/http/util/NamedObjectNotFoundException.java new file mode 100644 index 0000000..1415beb --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/NamedObjectNotFoundException.java @@ -0,0 +1,14 @@ +package org.xbib.elx.http.util; + +import org.elasticsearch.common.xcontent.XContentLocation; + +public class NamedObjectNotFoundException extends XContentParseException { + + public NamedObjectNotFoundException(String message) { + this(null, message); + } + + public NamedObjectNotFoundException(XContentLocation location, String message) { + super(location, message); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/NamedXContentRegistry.java b/elx-http/src/main/java/org/xbib/elx/http/util/NamedXContentRegistry.java new file mode 100644 index 0000000..ad5362c --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/NamedXContentRegistry.java @@ -0,0 +1,101 @@ +package org.xbib.elx.http.util; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.unmodifiableMap; + +public class NamedXContentRegistry { + + public static class Entry { + /** The class that this entry can read. */ + public final Class categoryClass; + + /** A name for the entry which is unique within the {@link #categoryClass}. */ + public final ParseField name; + + /** A parser capability of parser the entry's class. */ + private final ContextParser parser; + + /** Creates a new entry which can be stored by the registry. */ + public Entry(Class categoryClass, ParseField name, CheckedFunction parser) { + this.categoryClass = Objects.requireNonNull(categoryClass); + this.name = Objects.requireNonNull(name); + this.parser = Objects.requireNonNull((p, c) -> parser.apply(p)); + } + /** + * Creates a new entry which can be stored by the registry. + * Prefer {@link Entry#Entry(Class, ParseField, CheckedFunction)} unless you need a context to carry around while parsing. + */ + public Entry(Class categoryClass, ParseField name, ContextParser parser) { + this.categoryClass = Objects.requireNonNull(categoryClass); + this.name = Objects.requireNonNull(name); + this.parser = Objects.requireNonNull(parser); + } + } + + private final Map, Map> registry; + + public NamedXContentRegistry(List entries) { + if (entries.isEmpty()) { + registry = emptyMap(); + return; + } + entries = new ArrayList<>(entries); + entries.sort(Comparator.comparing(e -> e.categoryClass.getName())); + + Map, Map> registry = new HashMap<>(); + Map parsers = null; + Class currentCategory = null; + for (Entry entry : entries) { + if (currentCategory != entry.categoryClass) { + if (currentCategory != null) { + // we've seen the last of this category, put it into the big map + registry.put(currentCategory, unmodifiableMap(parsers)); + } + parsers = new HashMap<>(); + currentCategory = entry.categoryClass; + } + + for (String name : entry.name.getAllNamesIncludedDeprecated()) { + Object old = parsers.put(name, entry); + if (old != null) { + throw new IllegalArgumentException("NamedXContent [" + currentCategory.getName() + "][" + entry.name + "]" + + " is already registered for [" + old.getClass().getName() + "]," + + " cannot register [" + entry.parser.getClass().getName() + "]"); + } + } + } + // handle the last category + registry.put(currentCategory, unmodifiableMap(parsers)); + + this.registry = unmodifiableMap(registry); + } + + public T parseNamedObject(Class categoryClass, String name, XContentParser parser, C context) throws IOException { + Map parsers = registry.get(categoryClass); + if (parsers == null) { + if (registry.isEmpty()) { + // The "empty" registry will never work so we throw a better exception as a hint. + throw new NamedObjectNotFoundException("named objects are not supported for this parser"); + } + throw new NamedObjectNotFoundException("unknown named object category [" + categoryClass.getName() + "]"); + } + Entry entry = parsers.get(name); + if (entry == null) { + throw new NamedObjectNotFoundException(parser.getTokenLocation(), "unable to parse " + categoryClass.getSimpleName() + + " with name [" + name + "]: parser not found"); + } + return categoryClass.cast(entry.parser.parse(parser, context)); + } + +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/ObjectParser.java b/elx-http/src/main/java/org/xbib/elx/http/util/ObjectParser.java new file mode 100644 index 0000000..febb64b --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/ObjectParser.java @@ -0,0 +1,441 @@ +package org.xbib.elx.http.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import static org.elasticsearch.common.xcontent.XContentParser.Token.START_ARRAY; +import static org.elasticsearch.common.xcontent.XContentParser.Token.START_OBJECT; +import static org.elasticsearch.common.xcontent.XContentParser.Token.VALUE_BOOLEAN; +import static org.elasticsearch.common.xcontent.XContentParser.Token.VALUE_EMBEDDED_OBJECT; +import static org.elasticsearch.common.xcontent.XContentParser.Token.VALUE_NULL; +import static org.elasticsearch.common.xcontent.XContentParser.Token.VALUE_NUMBER; +import static org.elasticsearch.common.xcontent.XContentParser.Token.VALUE_STRING; + +/** + * A declarative, stateless parser that turns XContent into setter calls. A single parser should be defined for each object being parsed, + * nested elements can be added via {@link #declareObject(BiConsumer, ContextParser, ParseField)} which should be satisfied where possible + * by passing another instance of {@link ObjectParser}, this one customized for that Object. + *

+ * This class works well for object that do have a constructor argument or that can be built using information available from earlier in the + * XContent. + *

+ *

+ * Instances of {@link ObjectParser} should be setup by declaring a constant field for the parsers and declaring all fields in a static + * block just below the creation of the parser. Like this: + *

+ * 
{@code
+ *   private static final ObjectParser PARSER = new ObjectParser<>("thing", Thing::new));
+ *   static {
+ *       PARSER.declareInt(Thing::setMineral, new ParseField("mineral"));
+ *       PARSER.declareInt(Thing::setFruit, new ParseField("fruit"));
+ *   }
+ * }
+ * It's highly recommended to use the high level declare methods like {@link #declareString(BiConsumer, ParseField)} instead of + * {@link #declareField} which can be used to implement exceptional parsing operations not covered by the high level methods. + */ +public final class ObjectParser extends AbstractObjectParser { + + private static final Logger logger = LogManager.getLogger(ObjectParser.class.getName()); + + public static BiConsumer> fromList(Class c, + BiConsumer consumer) { + return (Value v, List l) -> { + @SuppressWarnings("unchecked") + ElementValue[] array = (ElementValue[]) Array.newInstance(c, l.size()); + consumer.accept(v, l.toArray(array)); + }; + } + + private final Map fieldParserMap = new HashMap<>(); + + private final String name; + + private final Supplier valueSupplier; + + /** + * Should this parser ignore unknown fields? This should generally be set to true only when parsing responses from external systems, + * never when parsing requests from users. + */ + private final boolean ignoreUnknownFields; + + /** + * Creates a new ObjectParser instance with a name. This name is used to reference the parser in exceptions and messages. + */ + public ObjectParser(String name) { + this(name, null); + } + + /** + * Creates a new ObjectParser instance which a name. + * @param name the parsers name, used to reference the parser in exceptions and messages. + * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser. + */ + public ObjectParser(String name, @Nullable Supplier valueSupplier) { + this(name, false, valueSupplier); + } + + /** + * Creates a new ObjectParser instance which a name. + * @param name the parsers name, used to reference the parser in exceptions and messages. + * @param ignoreUnknownFields Should this parser ignore unknown fields? This should generally be set to true only when parsing + * responses from external systems, never when parsing requests from users. + * @param valueSupplier a supplier that creates a new Value instance used when the parser is used as an inner object parser. + */ + public ObjectParser(String name, boolean ignoreUnknownFields, @Nullable Supplier valueSupplier) { + this.name = name; + this.valueSupplier = valueSupplier; + this.ignoreUnknownFields = ignoreUnknownFields; + } + + /** + * Parses a Value from the given {@link XContentParser} + * @param parser the parser to build a value from + * @param context context needed for parsing + * @return a new value instance drawn from the provided value supplier on {@link #ObjectParser(String, Supplier)} + * @throws IOException if an IOException occurs. + */ + @Override + public Value parse(XContentParser parser, Context context) throws IOException { + if (valueSupplier == null) { + throw new NullPointerException("valueSupplier is not set"); + } + return parse(parser, valueSupplier.get(), context); + } + + /** + * Parses a Value from the given {@link XContentParser} + * @param parser the parser to build a value from + * @param value the value to fill from the parser + * @param context a context that is passed along to all declared field parsers + * @return the parsed value + * @throws IOException if an IOException occurs. + */ + public Value parse(XContentParser parser, Value value, Context context) throws IOException { + logger.debug("parse"); + XContentParser.Token token; + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + token = parser.currentToken(); + } else { + token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new XContentParseException(parser.getTokenLocation(), "[" + name + "] Expected START_OBJECT but was: " + token); + } + } + FieldParser fieldParser = null; + String currentFieldName = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + fieldParser = getParser(currentFieldName, parser); + logger.debug("currentFieldName={} fieldParser={}", currentFieldName, fieldParser); + } else { + if (currentFieldName == null) { + throw new XContentParseException(parser.getTokenLocation(), "[" + name + "] no field found"); + } + if (fieldParser == null) { + assert ignoreUnknownFields : "this should only be possible if configured to ignore known fields"; + parser.skipChildren(); // noop if parser points to a value, skips children if parser is start object or start array + } else { + fieldParser.assertSupports(name, parser, currentFieldName); + parseSub(parser, fieldParser, currentFieldName, value, context); + } + fieldParser = null; + } + } + return value; + } + + @Override + public Value apply(XContentParser parser, Context context) { + if (valueSupplier == null) { + throw new NullPointerException("valueSupplier is not set"); + } + try { + return parse(parser, valueSupplier.get(), context); + } catch (IOException e) { + throw new XContentParseException(parser.getTokenLocation(), "[" + name + "] failed to parse object", e); + } + } + + public interface Parser { + void parse(XContentParser parser, Value value, Context context) throws IOException; + } + + public void declareField(Parser p, ParseField parseField, ValueType type) { + if (parseField == null) { + throw new IllegalArgumentException("[parseField] is required"); + } + if (type == null) { + throw new IllegalArgumentException("[type] is required"); + } + FieldParser fieldParser = new FieldParser(p, type.supportedTokens(), parseField, type); + for (String fieldValue : parseField.getAllNamesIncludedDeprecated()) { + fieldParserMap.putIfAbsent(fieldValue, fieldParser); + } + } + + @Override + public void declareField(BiConsumer consumer, ContextParser parser, ParseField parseField, + ValueType type) { + if (consumer == null) { + throw new IllegalArgumentException("[consumer] is required"); + } + if (parser == null) { + throw new IllegalArgumentException("[parser] is required"); + } + declareField((p, v, c) -> consumer.accept(v, parser.parse(p, c)), parseField, type); + } + + public void declareObjectOrDefault(BiConsumer consumer, BiFunction objectParser, + Supplier defaultValue, ParseField field) { + declareField((p, v, c) -> { + if (p.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { + if (p.booleanValue()) { + consumer.accept(v, defaultValue.get()); + } + } else { + consumer.accept(v, objectParser.apply(p, c)); + } + }, field, ValueType.OBJECT_OR_BOOLEAN); + } + + @Override + public void declareNamedObjects(BiConsumer> consumer, NamedObjectParser namedObjectParser, + Consumer orderedModeCallback, ParseField field) { + // This creates and parses the named object + BiFunction objectParser = (XContentParser p, Context c) -> { + if (p.currentToken() != XContentParser.Token.FIELD_NAME) { + throw new XContentParseException(p.getTokenLocation(), "[" + field + "] can be a single object with any number of " + + "fields or an array where each entry is an object with a single field"); + } + // This messy exception nesting has the nice side effect of telling the use which field failed to parse + try { + String name = p.currentName(); + try { + return namedObjectParser.parse(p, c, name); + } catch (Exception e) { + throw new XContentParseException(p.getTokenLocation(), "[" + field + "] failed to parse field [" + name + "]", e); + } + } catch (IOException e) { + throw new XContentParseException(p.getTokenLocation(), "[" + field + "] error while parsing", e); + } + }; + declareField((XContentParser p, Value v, Context c) -> { + List fields = new ArrayList<>(); + XContentParser.Token token; + if (p.currentToken() == XContentParser.Token.START_OBJECT) { + // Fields are just named entries in a single object + while ((token = p.nextToken()) != XContentParser.Token.END_OBJECT) { + fields.add(objectParser.apply(p, c)); + } + } else if (p.currentToken() == XContentParser.Token.START_ARRAY) { + // Fields are objects in an array. Each object contains a named field. + orderedModeCallback.accept(v); + while ((token = p.nextToken()) != XContentParser.Token.END_ARRAY) { + if (token != XContentParser.Token.START_OBJECT) { + throw new XContentParseException(p.getTokenLocation(), "[" + field + "] can be a single object with any number of " + + "fields or an array where each entry is an object with a single field"); + } + p.nextToken(); // Move to the first field in the object + fields.add(objectParser.apply(p, c)); + p.nextToken(); // Move past the object, should be back to into the array + if (p.currentToken() != XContentParser.Token.END_OBJECT) { + throw new XContentParseException(p.getTokenLocation(), "[" + field + "] can be a single object with any number of " + + "fields or an array where each entry is an object with a single field"); + } + } + } + consumer.accept(v, fields); + }, field, ValueType.OBJECT_ARRAY); + } + + @Override + public void declareNamedObjects(BiConsumer> consumer, NamedObjectParser namedObjectParser, + ParseField field) { + Consumer orderedModeCallback = (v) -> { + throw new IllegalArgumentException("[" + field + "] doesn't support arrays. Use a single object with multiple fields."); + }; + declareNamedObjects(consumer, namedObjectParser, orderedModeCallback, field); + } + + /** + * Functional interface for instantiating and parsing named objects. See ObjectParserTests#NamedObject for the canonical way to + * implement this for objects that themselves have a parser. + */ + @FunctionalInterface + public interface NamedObjectParser { + T parse(XContentParser p, Context c, String name) throws IOException; + } + + /** + * Get the name of the parser. + */ + @Override + public String getName() { + return name; + } + + private void parseArray(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context) + throws IOException { + assert parser.currentToken() == XContentParser.Token.START_ARRAY : "Token was: " + parser.currentToken(); + parseValue(parser, fieldParser, currentFieldName, value, context); + } + + private void parseValue(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context) + throws IOException { + try { + fieldParser.parser.parse(parser, value, context); + } catch (Exception ex) { + throw new XContentParseException(parser.getTokenLocation(), + "[" + name + "] failed to parse field [" + currentFieldName + "]", ex); + } + } + + private void parseSub(XContentParser parser, FieldParser fieldParser, String currentFieldName, Value value, Context context) + throws IOException { + final XContentParser.Token token = parser.currentToken(); + switch (token) { + case START_OBJECT: + parseValue(parser, fieldParser, currentFieldName, value, context); + /* + * Well behaving parsers should consume the entire object but + * asserting that they do that is not something we can do + * efficiently here. Instead we can check that they end on an + * END_OBJECT. They could end on the *wrong* end object and + * this test won't catch them, but that is the price that we pay + * for having a cheap test. + */ + if (parser.currentToken() != XContentParser.Token.END_OBJECT) { + throw new IllegalStateException("parser for [" + currentFieldName + "] did not end on END_OBJECT"); + } + break; + case START_ARRAY: + parseArray(parser, fieldParser, currentFieldName, value, context); + /* + * Well behaving parsers should consume the entire array but + * asserting that they do that is not something we can do + * efficiently here. Instead we can check that they end on an + * END_ARRAY. They could end on the *wrong* end array and + * this test won't catch them, but that is the price that we pay + * for having a cheap test. + */ + if (parser.currentToken() != XContentParser.Token.END_ARRAY) { + throw new IllegalStateException("parser for [" + currentFieldName + "] did not end on END_ARRAY"); + } + break; + case END_OBJECT: + case END_ARRAY: + case FIELD_NAME: + throw new XContentParseException(parser.getTokenLocation(), "[" + name + "]" + token + " is unexpected"); + case VALUE_STRING: + case VALUE_NUMBER: + case VALUE_BOOLEAN: + case VALUE_EMBEDDED_OBJECT: + case VALUE_NULL: + parseValue(parser, fieldParser, currentFieldName, value, context); + } + } + + private FieldParser getParser(String fieldName, XContentParser xContentParser) { + FieldParser parser = fieldParserMap.get(fieldName); + if (parser == null && false == ignoreUnknownFields) { + throw new XContentParseException(xContentParser.getTokenLocation(), + "[" + name + "] unknown field [" + fieldName + "], parser not found"); + } + return parser; + } + + private class FieldParser { + private final Parser parser; + private final EnumSet supportedTokens; + private final ParseField parseField; + private final ValueType type; + + FieldParser(Parser parser, EnumSet supportedTokens, ParseField parseField, ValueType type) { + this.parser = parser; + this.supportedTokens = supportedTokens; + this.parseField = parseField; + this.type = type; + } + + void assertSupports(String parserName, XContentParser parser, String currentFieldName) { + if (!supportedTokens.contains(parser.currentToken())) { + throw new XContentParseException(parser.getTokenLocation(), + "[" + parserName + "] " + currentFieldName + " doesn't support values of type: " + parser.currentToken()); + } + } + + @Override + public String toString() { + return "FieldParser{" + + "preferred_name=" + parseField.getPreferredName() + + ", supportedTokens=" + supportedTokens + + ", type=" + type.name() + + '}'; + } + } + + public enum ValueType { + STRING(VALUE_STRING), + STRING_OR_NULL(VALUE_STRING, VALUE_NULL), + FLOAT(VALUE_NUMBER, VALUE_STRING), + FLOAT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL), + DOUBLE(VALUE_NUMBER, VALUE_STRING), + DOUBLE_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL), + LONG(VALUE_NUMBER, VALUE_STRING), + LONG_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL), + INT(VALUE_NUMBER, VALUE_STRING), + INT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL), + BOOLEAN(VALUE_BOOLEAN, VALUE_STRING), + STRING_ARRAY(START_ARRAY, VALUE_STRING), + FLOAT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), + DOUBLE_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), + LONG_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), + INT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), + BOOLEAN_ARRAY(START_ARRAY, VALUE_BOOLEAN), + OBJECT(START_OBJECT), + OBJECT_OR_NULL(START_OBJECT, VALUE_NULL), + OBJECT_ARRAY(START_OBJECT, START_ARRAY), + OBJECT_OR_BOOLEAN(START_OBJECT, VALUE_BOOLEAN), + OBJECT_OR_STRING(START_OBJECT, VALUE_STRING), + OBJECT_OR_LONG(START_OBJECT, VALUE_NUMBER), + OBJECT_ARRAY_BOOLEAN_OR_STRING(START_OBJECT, START_ARRAY, VALUE_BOOLEAN, VALUE_STRING), + OBJECT_ARRAY_OR_STRING(START_OBJECT, START_ARRAY, VALUE_STRING), + VALUE(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING), + VALUE_OBJECT_ARRAY(VALUE_BOOLEAN, VALUE_NULL, VALUE_EMBEDDED_OBJECT, VALUE_NUMBER, VALUE_STRING, START_OBJECT, START_ARRAY), + VALUE_ARRAY(VALUE_BOOLEAN, VALUE_NULL, VALUE_NUMBER, VALUE_STRING, START_ARRAY); + + private final EnumSet tokens; + + ValueType(XContentParser.Token first, XContentParser.Token... rest) { + this.tokens = EnumSet.of(first, rest); + } + + public EnumSet supportedTokens() { + return this.tokens; + } + } + + @Override + public String toString() { + return "ObjectParser{" + + "name='" + name + '\'' + + ", fields=" + fieldParserMap.values() + + '}'; + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/XContentParseException.java b/elx-http/src/main/java/org/xbib/elx/http/util/XContentParseException.java new file mode 100644 index 0000000..254179f --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/XContentParseException.java @@ -0,0 +1,47 @@ +package org.xbib.elx.http.util; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.xcontent.XContentLocation; + +import java.util.Optional; + +/** + * Thrown when one of the XContent parsers cannot parse something. + */ +public class XContentParseException extends IllegalArgumentException { + + private final Optional location; + + public XContentParseException(String message) { + this(null, message); + } + + public XContentParseException(XContentLocation location, String message) { + super(message); + this.location = Optional.ofNullable(location); + } + + public XContentParseException(XContentLocation location, String message, Exception cause) { + super(message, cause); + this.location = Optional.ofNullable(location); + } + + public int getLineNumber() { + return location.map(l -> l.lineNumber).orElse(-1); + } + + public int getColumnNumber() { + return location.map(l -> l.columnNumber).orElse(-1); + } + + @Nullable + public XContentLocation getLocation() { + return location.orElse(null); + } + + @Override + public String getMessage() { + return location.map(l -> "[" + l.toString() + "] ").orElse("") + super.getMessage(); + } + +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/XContentParserUtils.java b/elx-http/src/main/java/org/xbib/elx/http/util/XContentParserUtils.java new file mode 100644 index 0000000..fb78890 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/XContentParserUtils.java @@ -0,0 +1,68 @@ +package org.xbib.elx.http.util; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentLocation; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.Aggregation; +import org.xbib.elx.http.util.aggregations.ParsedStringTerms; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public class XContentParserUtils { + + private static final NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(getDefaultNamedXContents()); + + public static void ensureExpectedToken(XContentParser.Token expected, XContentParser.Token actual, Supplier location) { + if (actual != expected) { + String message = "Failed to parse object: expecting token of type [%s] but found [%s]"; + throw new ElasticsearchException(location.get() + ":" + String.format(Locale.ROOT, message, expected, actual)); + } + } + + public static void parseTypedKeysObject(XContentParser parser, String delimiter, Class objectClass, Consumer consumer) + throws IOException { + if (parser.currentToken() != XContentParser.Token.START_OBJECT && parser.currentToken() != XContentParser.Token.START_ARRAY) { + throwUnknownToken(parser.currentToken(), parser.getTokenLocation()); + } + String currentFieldName = parser.currentName(); + if (Strings.hasLength(currentFieldName)) { + int position = currentFieldName.indexOf(delimiter); + if (position > 0) { + String type = currentFieldName.substring(0, position); + String name = currentFieldName.substring(position + 1); + consumer.accept(namedObject(parser, objectClass, type, name)); + return; + } + // if we didn't find a delimiter we ignore the object or array for forward compatibility instead of throwing an error + parser.skipChildren(); + } else { + throw new ElasticsearchException(parser.getTokenLocation() + ":" + "Failed to parse object: empty key"); + } + } + + public static void throwUnknownToken(XContentParser.Token token, XContentLocation location) { + String message = "Failed to parse object: unexpected token [%s] found"; + throw new ElasticsearchException(location + ":" + String.format(Locale.ROOT, message, token)); + } + + static T namedObject(XContentParser parser, Class categoryClass, String name, Object context) throws IOException { + return xContentRegistry.parseNamedObject(categoryClass, name, parser, context); + } + + public static List getDefaultNamedXContents() { + Map> map = new HashMap<>(); + //map.put("terms", (p, c) -> ParsedStringTerms.fromXContent(p, (String) c)); + return map.entrySet().stream() + .map(entry -> new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(entry.getKey()), entry.getValue())) + .collect(Collectors.toList()); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/CommonFields.java b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/CommonFields.java new file mode 100644 index 0000000..27c1c55 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/CommonFields.java @@ -0,0 +1,18 @@ +package org.xbib.elx.http.util.aggregations; + +import org.elasticsearch.common.ParseField; + +final class CommonFields { + public static final ParseField META = new ParseField("meta"); + public static final ParseField BUCKETS = new ParseField("buckets"); + public static final ParseField VALUE = new ParseField("value"); + public static final ParseField VALUES = new ParseField("values"); + public static final ParseField VALUE_AS_STRING = new ParseField("value_as_string"); + public static final ParseField DOC_COUNT = new ParseField("doc_count"); + public static final ParseField KEY = new ParseField("key"); + public static final ParseField KEY_AS_STRING = new ParseField("key_as_string"); + public static final ParseField FROM = new ParseField("from"); + public static final ParseField FROM_AS_STRING = new ParseField("from_as_string"); + public static final ParseField TO = new ParseField("to"); + public static final ParseField TO_AS_STRING = new ParseField("to_as_string"); +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedAggregation.java b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedAggregation.java new file mode 100644 index 0000000..b110aa6 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedAggregation.java @@ -0,0 +1,40 @@ +package org.xbib.elx.http.util.aggregations; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.search.aggregations.Aggregation; +import org.xbib.elx.http.util.ObjectParser; + +import java.util.Collections; +import java.util.Map; + +/** + * An implementation of {@link Aggregation} that is parsed from a REST response. + * Serves as a base class for all aggregation implementations that are parsed from REST. + */ +public abstract class ParsedAggregation implements Aggregation { + + protected static void declareAggregationFields(ObjectParser objectParser) { + objectParser.declareObject((parsedAgg, metadata) -> parsedAgg.metadata = Collections.unmodifiableMap(metadata), + (parser, context) -> parser.map(), CommonFields.META); + } + + private String name; + protected Map metadata; + + @Override + public final String getName() { + return name; + } + + protected void setName(String name) { + this.name = name; + } + + @Override + public final Map getMetaData() { + return metadata; + } +} \ No newline at end of file diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedMultiBucketAggregation.java b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedMultiBucketAggregation.java new file mode 100644 index 0000000..bd0c81d --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedMultiBucketAggregation.java @@ -0,0 +1,149 @@ +package org.xbib.elx.http.util.aggregations; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; +import org.xbib.elx.http.util.CheckedBiConsumer; +import org.xbib.elx.http.util.CheckedFunction; +import org.xbib.elx.http.util.ObjectParser; +import org.xbib.elx.http.util.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.xbib.elx.http.util.XContentParserUtils.ensureExpectedToken; + +public abstract class ParsedMultiBucketAggregation + extends ParsedAggregation implements MultiBucketsAggregation { + + protected final List buckets = new ArrayList<>(); + + protected boolean keyed = false; + + protected static void declareMultiBucketAggregationFields(final ObjectParser objectParser, + final CheckedFunction bucketParser, + final CheckedFunction keyedBucketParser) { + declareAggregationFields(objectParser); + objectParser.declareField((parser, aggregation, context) -> { + XContentParser.Token token = parser.currentToken(); + if (token == XContentParser.Token.START_OBJECT) { + aggregation.keyed = true; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + aggregation.buckets.add(keyedBucketParser.apply(parser)); + } + } else if (token == XContentParser.Token.START_ARRAY) { + aggregation.keyed = false; + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + aggregation.buckets.add(bucketParser.apply(parser)); + } + } + }, CommonFields.BUCKETS, ObjectParser.ValueType.OBJECT_ARRAY); + } + + public abstract static class ParsedBucket implements MultiBucketsAggregation.Bucket { + + private Aggregations aggregations; + private String keyAsString; + private long docCount; + private boolean keyed; + + protected void setKeyAsString(String keyAsString) { + this.keyAsString = keyAsString; + } + + @Override + public String getKeyAsString() { + return keyAsString; + } + + protected void setDocCount(long docCount) { + this.docCount = docCount; + } + + @Override + public long getDocCount() { + return docCount; + } + + public void setKeyed(boolean keyed) { + this.keyed = keyed; + } + + protected boolean isKeyed() { + return keyed; + } + + protected void setAggregations(Aggregations aggregations) { + this.aggregations = aggregations; + } + + @Override + public Aggregations getAggregations() { + return aggregations; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + /*if (keyed) { + builder.startObject(getKeyAsString()); + } else { + builder.startObject(); + } + if (keyAsString != null) { + builder.field(CommonFields.KEY_AS_STRING.getPreferredName(), getKeyAsString()); + } + keyToXContent(builder); + builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); + aggregations.toXContentInternal(builder, params); + builder.endObject();*/ + return builder; + } + + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), getKey()); + } + + protected static B parseXContent(final XContentParser parser, + final boolean keyed, + final Supplier bucketSupplier, + final CheckedBiConsumer keyConsumer) + throws IOException { + final B bucket = bucketSupplier.get(); + bucket.setKeyed(keyed); + XContentParser.Token token = parser.currentToken(); + String currentFieldName = parser.currentName(); + if (keyed) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, parser::getTokenLocation); + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + } + List aggregations = new ArrayList<>(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.setKeyAsString(parser.text()); + } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + keyConsumer.accept(parser, bucket); + } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + bucket.setDocCount(parser.longValue()); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + keyConsumer.accept(parser, bucket); + } else { + XContentParserUtils.parseTypedKeysObject(parser, "#", InternalAggregation.class, + aggregations::add); + } + } + } + bucket.setAggregations(new InternalAggregations(aggregations)); + return bucket; + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedStringTerms.java b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedStringTerms.java new file mode 100644 index 0000000..b2f759b --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedStringTerms.java @@ -0,0 +1,103 @@ +package org.xbib.elx.http.util.aggregations; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.xbib.elx.http.util.ObjectParser; + +import java.io.IOException; +import java.nio.CharBuffer; +import java.util.List; + +public class ParsedStringTerms extends ParsedTerms { + + public String getType() { + return "terms"; + } + + private static ObjectParser PARSER = + new ObjectParser<>(ParsedStringTerms.class.getSimpleName(), true, ParsedStringTerms::new); + + static { + declareParsedTermsFields(PARSER, ParsedBucket::fromXContent); + } + + public static ParsedStringTerms fromXContent(XContentParser parser, String name) throws IOException { + ParsedStringTerms aggregation = PARSER.parse(parser, null); + aggregation.setName(name); + return aggregation; + } + + @Override + public Object getProperty(String path) { + throw new UnsupportedOperationException(); + } + + public static class ParsedBucket extends ParsedTerms.ParsedBucket { + + private BytesRef key; + + @Override + public Object getKey() { + return getKeyAsString(); + } + + @Override + public String getKeyAsString() { + String keyAsString = super.getKeyAsString(); + if (keyAsString != null) { + return keyAsString; + } + if (key != null) { + return key.utf8ToString(); + } + return null; + } + + @Override + public Object getProperty(String containingAggName, List path) { + throw new UnsupportedOperationException(); + } + + public Number getKeyAsNumber() { + if (key != null) { + return Double.parseDouble(key.utf8ToString()); + } + return null; + } + + protected XContentBuilder keyToXContent(XContentBuilder builder) throws IOException { + return builder.field(CommonFields.KEY.getPreferredName(), getKey()); + } + + static ParsedBucket fromXContent(XContentParser parser) throws IOException { + return parseTermsBucketXContent(parser, ParsedBucket::new, (p, bucket) -> { + CharBuffer cb = charBufferOrNull(p); + if (cb == null) { + bucket.key = null; + } else { + bucket.key = new BytesRef(cb); + } + }); + } + + static CharBuffer charBufferOrNull(XContentParser parser) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { + return null; + } + return CharBuffer.wrap(parser.textCharacters(), parser.textOffset(), parser.textLength()); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedTerms.java b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedTerms.java new file mode 100644 index 0000000..fc34516 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/util/aggregations/ParsedTerms.java @@ -0,0 +1,118 @@ +package org.xbib.elx.http.util.aggregations; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.InternalAggregations; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.xbib.elx.http.util.CheckedBiConsumer; +import org.xbib.elx.http.util.CheckedFunction; +import org.xbib.elx.http.util.ObjectParser; +import org.xbib.elx.http.util.XContentParserUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +public abstract class ParsedTerms extends ParsedMultiBucketAggregation implements Terms { + + protected static final ParseField DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME = new ParseField("doc_count_error_upper_bound"); + + protected static final ParseField SUM_OF_OTHER_DOC_COUNTS = new ParseField("sum_other_doc_count"); + + protected long docCountErrorUpperBound; + + protected long sumOtherDocCount; + + @Override + public long getDocCountError() { + return docCountErrorUpperBound; + } + + @Override + public long getSumOfOtherDocCounts() { + return sumOtherDocCount; + } + + @Override + public List getBuckets() { + //return buckets; + throw new UnsupportedOperationException(); + } + + @Override + public Terms.Bucket getBucketByKey(String term) { + for (Terms.Bucket bucket : getBuckets()) { + if (bucket.getKeyAsString().equals(term)) { + return bucket; + } + } + return null; + } + + static void declareParsedTermsFields(final ObjectParser objectParser, + final CheckedFunction bucketParser) { + declareMultiBucketAggregationFields(objectParser, bucketParser::apply, bucketParser::apply); + objectParser.declareLong((parsedTerms, value) -> parsedTerms.docCountErrorUpperBound = value , + DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME); + objectParser.declareLong((parsedTerms, value) -> parsedTerms.sumOtherDocCount = value, + SUM_OF_OTHER_DOC_COUNTS); + } + + public abstract static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket /*implements Terms.Bucket*/ { + + boolean showDocCountError = false; + protected long docCountError; + + public long getDocCountError() { + return docCountError; + } + + @Override + public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + /*builder.startObject(); + keyToXContent(builder); + builder.field(CommonFields.DOC_COUNT.getPreferredName(), getDocCount()); + if (showDocCountError) { + builder.field(DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName(), getDocCountError()); + } + getAggregations().toXContentInternal(builder, params); + builder.endObject();*/ + return builder; + } + + static B parseTermsBucketXContent(final XContentParser parser, final Supplier bucketSupplier, + final CheckedBiConsumer keyConsumer) + throws IOException { + + final B bucket = bucketSupplier.get(); + final List aggregations = new ArrayList<>(); + XContentParser.Token token; + String currentFieldName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else if (token.isValue()) { + if (CommonFields.KEY_AS_STRING.getPreferredName().equals(currentFieldName)) { + bucket.setKeyAsString(parser.text()); + } else if (CommonFields.KEY.getPreferredName().equals(currentFieldName)) { + keyConsumer.accept(parser, bucket); + } else if (CommonFields.DOC_COUNT.getPreferredName().equals(currentFieldName)) { + bucket.setDocCount(parser.longValue()); + } else if (DOC_COUNT_ERROR_UPPER_BOUND_FIELD_NAME.getPreferredName().equals(currentFieldName)) { + bucket.docCountError = parser.longValue(); + bucket.showDocCountError = true; + } + } else if (token == XContentParser.Token.START_OBJECT) { + XContentParserUtils.parseTypedKeysObject(parser, "#", InternalAggregation.class, + aggregations::add); + } + } + bucket.setAggregations(new InternalAggregations(aggregations)); + return bucket; + } + } +} diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider new file mode 100644 index 0000000..0c75f14 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.ExtendedHttpClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction new file mode 100644 index 0000000..3d3ea95 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -0,0 +1,3 @@ +org.xbib.elx.http.action.search.HttpSearchAction +org.xbib.elx.http.action.get.HttpGetAction +org.xbib.elx.http.action.get.HttpMultiGetAction diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java new file mode 100644 index 0000000..36f038a --- /dev/null +++ b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java @@ -0,0 +1,122 @@ +package org.xbib.elx.http.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetAction; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.index.IndexAction; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.junit.Ignore; +import org.junit.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.http.ExtendedHttpClient; +import org.xbib.elx.http.ExtendedHttpClientProvider; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ClientTest extends TestBase { + + private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); + + @Ignore + @Test + public void testGet() throws Exception { + try (ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put("url", "http://" + host + ":" + httpPort) + .build()) { + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index("test"); + indexRequest.type("test"); + indexRequest.id("1"); + indexRequest.source("test", "Hello Jörg"); + IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); + client("1").execute(RefreshAction.INSTANCE, new RefreshRequest()); + + GetRequest getRequest = new GetRequest(); + getRequest.index("test"); + getRequest.type("test"); + getRequest.id("1"); + + GetResponse getResponse = client.execute(GetAction.INSTANCE, getRequest).actionGet(); + + assertTrue(getResponse.isExists()); + assertEquals("{\"test\":\"Hello Jörg\"}", getResponse.getSourceAsString()); + + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } + } + + @Ignore + @Test + public void testMultiGet() throws Exception { + try (ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put("url", "http://" + host + ":" + httpPort) + .build()) { + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index("test"); + indexRequest.type("test"); + indexRequest.id("1"); + indexRequest.source("test", "Hello Jörg"); + IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); + client("1").execute(RefreshAction.INSTANCE, new RefreshRequest()); + + MultiGetRequest multiGetRequest = new MultiGetRequest(); + multiGetRequest.add("test", "test", "1"); + + MultiGetResponse multiGetResponse = client.execute(MultiGetAction.INSTANCE, multiGetRequest).actionGet(); + + assertEquals(1, multiGetResponse.getResponses().length); + assertEquals("{\"test\":\"Hello Jörg\"}", multiGetResponse.getResponses()[0].getResponse().getSourceAsString()); + + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } + } + + @Test + public void testSearchDoc() throws Exception { + try (ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put("url", "http://" + host + ":" + httpPort) + .build()) { + IndexRequest indexRequest = new IndexRequest(); + indexRequest.index("test"); + indexRequest.type("test"); + indexRequest.id("1"); + indexRequest.source("test", "Hello Jörg"); + IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); + client("1").execute(RefreshAction.INSTANCE, new RefreshRequest()); + + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(QueryBuilders.matchAllQuery()); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices("test"); + searchRequest.types("test"); + searchRequest.source(builder); + SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); + long hits = searchResponse.getHits().getTotalHits(); + assertEquals(1, hits); + logger.info("hits = {} source = {}", hits, searchResponse.getHits().getHits()[0].getSourceAsString()); + assertEquals("{\"test\":\"Hello Jörg\"}", searchResponse.getHits().getHits()[0].getSourceAsString()); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java b/elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java similarity index 85% rename from elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java rename to elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java index 747e333..a344fc7 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/MockNode.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/MockNode.java @@ -1,4 +1,4 @@ -package org.xbib.elx.transport; +package org.xbib.elx.http.test; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.node.Node; @@ -8,4 +8,5 @@ public class MockNode extends Node { public MockNode(Settings settings) { super(settings); } + } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/TestBase.java b/elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java similarity index 86% rename from elx-transport/src/test/java/org/xbib/elx/transport/TestBase.java rename to elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java index 95cedb4..78a6485 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/TestBase.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/TestBase.java @@ -1,4 +1,4 @@ -package org.xbib.elx.transport; +package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -6,6 +6,7 @@ import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; @@ -15,6 +16,7 @@ import org.elasticsearch.client.support.AbstractClient; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; import org.junit.After; @@ -47,9 +49,11 @@ public class TestBase { private String cluster; - private String host; + protected String host; - private int port; + protected int port; + + protected int httpPort; @Before public void startNodes() { @@ -60,7 +64,7 @@ public class TestBase { findNodeAddress(); try { ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.YELLOW) .timeout(TimeValue.timeValueSeconds(30))).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { throw new IOException("cluster state is " + healthResponse.getStatus().name() @@ -119,14 +123,6 @@ public class TestBase { protected Settings getNodeSettings() { return settingsBuilder() .put("cluster.name", cluster) - //.put("cluster.routing.schedule", "50ms") - //.put("cluster.routing.allocation.disk.threshold_enabled", false) - //.put("discovery.zen.multicast.enabled", true) - //.put("discovery.zen.multicast.ping_timeout", "5s") - //.put("http.enabled", true) - //.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) - //.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low - //.put("index.number_of_replicas", 0) .put("path.home", getHome()) .build(); } @@ -146,12 +142,18 @@ public class TestBase { protected void findNodeAddress() { NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - Object obj = response.iterator().next().getTransport().getAddress() - .publishAddress(); - if (obj instanceof InetSocketTransportAddress) { - InetSocketTransportAddress address = (InetSocketTransportAddress) obj; - host = address.address().getHostName(); - port = address.address().getPort(); + for (NodeInfo nodeInfo : response) { + TransportAddress transportAddress = nodeInfo.getTransport().getAddress().publishAddress(); + if (transportAddress instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) transportAddress; + host = address.address().getHostName(); + port = address.address().getPort(); + } + transportAddress = nodeInfo.getHttp().getAddress().publishAddress(); + if (transportAddress instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) transportAddress; + httpPort = address.address().getPort(); + } } } diff --git a/elx-http/src/test/resources/log4j2.xml b/elx-http/src/test/resources/log4j2.xml new file mode 100644 index 0000000..1258d7f --- /dev/null +++ b/elx-http/src/test/resources/log4j2.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java index dc147b0..f922bcc 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java @@ -1,10 +1,5 @@ package org.xbib.elx.node.test; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; @@ -17,9 +12,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.node.ExtendedNodeClient; @@ -29,34 +25,70 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -public class ClientTest extends TestBase { +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class ClientTest { private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - private static final Long ACTIONS = 25000L; + private static final Long ACTIONS = 1000L; - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 100L; - @Before - public void startNodes() { - try { - super.startNodes(); - startNode("2"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } + private final TestExtension.Helper helper; + + ClientTest(TestExtension.Helper helper) { + this.helper = helper; } @Test - public void testSingleDoc() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testNewIndex() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + client.newIndex("test1"); + client.close(); + } + + @Test + void testMapping() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) + .provider(ExtendedNodeClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + client.newIndex("test2", Settings.EMPTY, builder.string()); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2"); + GetMappingsResponse getMappingsResponse = + client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); + logger.info("mappings={}", getMappingsResponse.getMappings()); + assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc")); + client.close(); + } + + @Test + void testSingleDoc() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) .build(); try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.newIndex("test3"); + client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { @@ -72,55 +104,21 @@ public class ClientTest extends TestBase { } @Test - public void testNewIndex() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - client.newIndex("test"); - client.close(); - } - - @Test - public void testMapping() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - XContentBuilder builder = jsonBuilder() - .startObject() - .startObject("doc") - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject() - .endObject(); - client.newIndex("test", Settings.EMPTY, builder.string()); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); - client.close(); - } - - @Test - public void testRandomDocs() throws Exception { + void testRandomDocs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test"); + client.newIndex("test4"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test4", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + client.waitForResponses(60L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { @@ -129,9 +127,11 @@ public class ClientTest extends TestBase { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); + client.refreshIndex("test4"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); + .setIndices("test4") + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0); assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.close(); @@ -139,37 +139,38 @@ public class ClientTest extends TestBase { } @Test - public void testThreadedRandomDocs() throws Exception { + void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final Long actions = ACTIONS; + long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final long actions = ACTIONS; logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test") - .startBulk("test", -1, 1000); - ThreadPoolExecutor pool = EsExecutors.newFixed("bulk-nodeclient-test", maxthreads, 30, - EsExecutors.daemonThreadFactory("bulk-nodeclient-test")); + client.newIndex("test5") + .startBulk("test5", -1, 1000); + ThreadPoolExecutor pool = EsExecutors.newFixed("nodeclient-test", maxthreads, 30, + EsExecutors.daemonThreadFactory("nodeclient-test")); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { pool.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false,"{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); } logger.info("waiting for latch..."); - if (latch.await(5, TimeUnit.MINUTES)) { + if (latch.await(60, TimeUnit.SECONDS)) { logger.info("flush..."); client.flush(); client.waitForResponses(60L, TimeUnit.SECONDS); - logger.info("got all responses, pool shutdown..."); + logger.info("pool shutdown..."); pool.shutdown(); logger.info("pool is shut down"); } else { @@ -178,15 +179,17 @@ public class ClientTest extends TestBase { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.stopBulk("test", 30L, TimeUnit.SECONDS); + client.stopBulk("test5", 60L, TimeUnit.SECONDS); assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); + client.refreshIndex("test5"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); + .setIndices("test5") + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0); assertEquals(maxthreads * actions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.close(); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index 9ea5c40..0ea421c 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -7,7 +7,8 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; import org.xbib.elx.node.ExtendedNodeClient; @@ -15,38 +16,47 @@ import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class DuplicateIDTest extends TestBase { +@ExtendWith(TestExtension.class) +class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 12345L; + private static final Long ACTIONS = 50L; + + private final TestExtension.Helper helper; + + DuplicateIDTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testDuplicateDocIDs() throws Exception { + void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { - client.newIndex("test"); + client.newIndex("test_dup"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test_dup", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test"); + client.refreshIndex("test_dup"); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.types("test"); + searchRequest.indices("test_dup"); searchRequest.source(builder); - long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); + long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index b0b6428..7a2c3fc 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -7,7 +7,8 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.node.ExtendedNodeClient; @@ -19,17 +20,24 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexPruneTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexPruneTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexPruneTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testPrune() throws IOException { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testPrune() throws IOException { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .build(); try { @@ -37,25 +45,22 @@ public class IndexPruneTest extends TestBase { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1", settings); - client.shiftIndex("test", "test1", Collections.emptyList()); - client.newIndex("test2", settings); - client.shiftIndex("test", "test2", Collections.emptyList()); - client.newIndex("test3", settings); - client.shiftIndex("test", "test3", Collections.emptyList()); - client.newIndex("test4", settings); - client.shiftIndex("test", "test4", Collections.emptyList()); - + client.newIndex("test_prune1", settings); + client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + client.newIndex("test_prune2", settings); + client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + client.newIndex("test_prune3", settings); + client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + client.newIndex("test_prune4", settings); + client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test", "test4", 2, 2, true); - - assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); - assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); - assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); - assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); - + client.pruneIndex("test_prune", "test_prune4", 2, 2, true); + assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); + assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); List list = new ArrayList<>(); - for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { + for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); indicesExistsRequest.indices(new String[] { index }); IndicesExistsResponse indicesExistsResponse = diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index 6c900e8..87d422c 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -7,7 +7,8 @@ import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.node.ExtendedNodeClient; @@ -17,16 +18,23 @@ import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexShiftTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexShiftTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexShiftTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testIndexShift() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void testIndexShift() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .build(); try { @@ -36,14 +44,13 @@ public class IndexShiftTest extends TestBase { .build(); client.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - IndexShiftResult indexShiftResult = - client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); - + client.shiftIndex("test_shift", "test1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); @@ -53,23 +60,24 @@ public class IndexShiftTest extends TestBase { assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("test")); + assertTrue(aliases.containsKey("test_shift")); - String resolved = client.resolveAlias("test"); + String resolved = client.resolveAlias("test_shift"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("test")); + assertTrue(aliases.containsKey("test_shift")); client.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + indexShiftResult = client.shiftIndex("test_shift", "test5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, index, alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -88,7 +96,7 @@ public class IndexShiftTest extends TestBase { assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - resolved = client.resolveAlias("test"); + resolved = client.resolveAlias("test_shift"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java deleted file mode 100644 index 78a83db..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ReplicaTest.java +++ /dev/null @@ -1,151 +0,0 @@ -package org.xbib.elx.node.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.indexing.IndexingStats; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -@Ignore -public class ReplicaTest extends TestBase { - - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName()); - - @Test - public void testReplicaLevel() throws Exception { - - // we need nodes for replica levels - startNode("2"); - startNode("3"); - startNode("4"); - - Settings settingsTest1 = Settings.settingsBuilder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 3) - .build(); - - Settings settingsTest2 = Settings.settingsBuilder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 1) - .build(); - - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - - try { - client.newIndex("test1", settingsTest1, new HashMap<>()) - .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 1234; i++) { - client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - for (int i = 0; i < 1234; i++) { - client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - logger.info("refreshing"); - client.refreshIndex("test1"); - client.refreshIndex("test2"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test1", "test2") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("query total hits={}", hits); - assertEquals(2468, hits); - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), IndicesStatsAction.INSTANCE) - .all(); - IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); - for (Map.Entry m : response.getIndices().entrySet()) { - IndexStats indexStats = m.getValue(); - CommonStats commonStats = indexStats.getTotal(); - IndexingStats indexingStats = commonStats.getIndexing(); - IndexingStats.Stats stats = indexingStats.getTotal(); - logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); - for (Map.Entry me : indexStats.getIndexShards().entrySet()) { - IndexShardStats indexShardStats = me.getValue(); - CommonStats commonShardStats = indexShardStats.getTotal(); - logger.info("shard {} count = {}", me.getKey(), - commonShardStats.getIndexing().getTotal().getIndexCount()); - } - } - try { - client.deleteIndex("test1") - .deleteIndex("test2"); - } catch (Exception e) { - logger.error("delete index failed, ignored. Reason:", e); - } - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - Settings settings = Settings.settingsBuilder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) - .provider(ExtendedNodeClientProvider.class) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 12345; i++) { - client.index("replicatest",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); - assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } - -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index 8000063..ae33bd9 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -4,7 +4,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.node.ExtendedNodeClient; @@ -12,47 +13,46 @@ import org.xbib.elx.node.ExtendedNodeClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -public class SmokeTest extends TestBase { +@ExtendWith(TestExtension.class) +class SmokeTest { private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); + private final TestExtension.Helper helper; + + SmokeTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void smokeTest() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(client("1")) + void smokeTest() throws Exception { + final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) .provider(ExtendedNodeClientProvider.class) .build(); try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.newIndex("test_smoke"); + client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - - assertEquals(getClusterName(), client.getClusterName()); - - client.checkMapping("test"); - - client.update("test", "1", "{ \"name\" : \"Another name\"}"); + assertEquals(helper.getCluster(), client.getClusterName()); + client.checkMapping("test_smoke"); + client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); client.flush(); - - client.waitForRecovery("test", 10L, TimeUnit.SECONDS); - - client.delete("test", "1"); - client.deleteIndex("test"); - - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test2", Settings.settingsBuilder() + client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); + client.delete("test_smoke", "1"); + client.deleteIndex("test_smoke"); + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.settingsBuilder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); client.newIndex(indexDefinition); client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.flush(); client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); assertEquals(0, client.getBulkMetric().getFailed().getCount()); assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java deleted file mode 100644 index 2c486e0..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/test/TestBase.java +++ /dev/null @@ -1,212 +0,0 @@ -package org.xbib.elx.node.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.support.AbstractClient; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.node.Node; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; - -public class TestBase { - - private static final Logger logger = LogManager.getLogger("test"); - - private static final Random random = new Random(); - - private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String cluster; - - private String host; - - private int port; - - @Before - public void startNodes() { - try { - logger.info("starting"); - setClusterName("test-cluster-" + System.getProperty("user.name")); - startNode("1"); - findNodeAddress(); - try { - ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) - .timeout(TimeValue.timeValueSeconds(30))).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - throw new IOException("cluster state is " + healthResponse.getStatus().name() - + ", from here on, everything will fail!"); - } - } catch (ElasticsearchTimeoutException e) { - throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); - } - ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); - ClusterStateResponse clusterStateResponse = - client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); - logger.info("host = {} port = {}", host, port); - - } catch (Throwable t) { - logger.error("startNodes failed", t); - } - } - - @After - public void stopNodes() { - try { - closeNodes(); - } catch (Exception e) { - logger.error("can not close nodes", e); - } finally { - try { - deleteFiles(); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes - } catch (IOException e) { - logger.error(e.getMessage(), e); - } catch (InterruptedException e) { - // ignore - } - } - } - - protected void setClusterName(String cluster) { - this.cluster = cluster; - } - - protected String getClusterName() { - return cluster; - } - - protected Settings getTransportSettings() { - return settingsBuilder() - .put("host", host) - .put("port", port) - .put("cluster.name", cluster) - .put("path.home", getHome()) - .build(); - } - - protected Settings getNodeSettings() { - return settingsBuilder() - .put("cluster.name", cluster) - //.put("cluster.routing.schedule", "50ms") - //.put("cluster.routing.allocation.disk.threshold_enabled", false) - //.put("discovery.zen.multicast.enabled", true) - //.put("discovery.zen.multicast.ping_timeout", "5s") - //.put("http.enabled", true) - //.put("threadpool.bulk.size", Runtime.getRuntime().availableProcessors()) - //.put("threadpool.bulk.queue_size", 16 * Runtime.getRuntime().availableProcessors()) // default is 50, too low - //.put("index.number_of_replicas", 0) - .put("path.home", getHome()) - .build(); - } - - protected static String getHome() { - return System.getProperty("path.home", System.getProperty("user.dir")); - } - - protected void startNode(String id) { - buildNode(id).start(); - } - - protected AbstractClient client(String id) { - return clients.get(id); - } - - protected void findNodeAddress() { - NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); - NodesInfoResponse response = client("1").admin().cluster().nodesInfo(nodesInfoRequest).actionGet(); - Object obj = response.iterator().next().getTransport().getAddress() - .publishAddress(); - if (obj instanceof InetSocketTransportAddress) { - InetSocketTransportAddress address = (InetSocketTransportAddress) obj; - host = address.address().getHostName(); - port = address.address().getPort(); - } - } - - private Node buildNode(String id) { - Settings nodeSettings = settingsBuilder() - .put(getNodeSettings()) - .put("name", id) - .build(); - Node node = new MockNode(nodeSettings); - AbstractClient client = (AbstractClient) node.client(); - nodes.put(id, node); - clients.put(id, client); - logger.info("clients={}", clients); - return node; - } - - protected String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - private void closeNodes() { - logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { - client.close(); - } - clients.clear(); - logger.info("closing all nodes"); - for (Node node : nodes.values()) { - if (node != null) { - node.close(); - } - } - nodes.clear(); - logger.info("all nodes closed"); - } - - private static void deleteFiles() throws IOException { - Path directory = Paths.get(getHome() + "/data"); - Files.walkFileTree(directory, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java new file mode 100644 index 0000000..7d28686 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java @@ -0,0 +1,213 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class TestExtension implements ParameterResolver, BeforeAllCallback, AfterAllCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private String home; + + private String cluster; + + private String host; + + private int port; + + private static final String key = "es-instance"; + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); + setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + setClusterName("test-cluster-" + System.getProperty("user.name")); + deleteFiles(Paths.get(getHome() + "/data")); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes + logger.info("starting cluster"); + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + closeNodes(); + deleteFiles(Paths.get(getHome() + "/data")); + } + + private void setClusterName(String cluster) { + this.cluster = cluster; + } + + private String getClusterName() { + return cluster; + } + + private void setHome(String home) { + this.home = home; + } + + private String getHome() { + return home; + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + return new Helper(); + } + + class Helper { + + void startNode(String id) { + buildNode(id).start(); + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .put("name", id) + .build(); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String getCluster() { + return getClusterName(); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java deleted file mode 100644 index c4f9af0..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ReplicaTest.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.xbib.elx.transport; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.stats.CommonStats; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndexStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.indexing.IndexingStats; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class ReplicaTest extends TestBase { - - private static final Logger logger = LogManager.getLogger(ReplicaTest.class.getName()); - - @Test - public void testReplicaLevel() throws Exception { - - // we need nodes for replica levels - startNode("2"); - startNode("3"); - startNode("4"); - - Settings settingsTest1 = Settings.settingsBuilder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 3) - .build(); - - Settings settingsTest2 = Settings.settingsBuilder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 1) - .build(); - - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) - .build(); - - try { - client.newIndex("test1", settingsTest1, new HashMap<>()) - .newIndex("test2", settingsTest2, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 1234; i++) { - client.index("test1", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - for (int i = 0; i < 1234; i++) { - client.index("test2", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test1"); - client.refreshIndex("test2"); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test1", "test2") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("query total hits={}", hits); - assertEquals(2468, hits); - - // TODO move to api - IndicesStatsRequestBuilder indicesStatsRequestBuilder = new IndicesStatsRequestBuilder(client.getClient(), - IndicesStatsAction.INSTANCE).all(); - IndicesStatsResponse response = indicesStatsRequestBuilder.execute().actionGet(); - for (Map.Entry m : response.getIndices().entrySet()) { - IndexStats indexStats = m.getValue(); - CommonStats commonStats = indexStats.getTotal(); - IndexingStats indexingStats = commonStats.getIndexing(); - IndexingStats.Stats stats = indexingStats.getTotal(); - logger.info("index {}: count = {}", m.getKey(), stats.getIndexCount()); - for (Map.Entry me : indexStats.getIndexShards().entrySet()) { - IndexShardStats indexShardStats = me.getValue(); - CommonStats commonShardStats = indexShardStats.getTotal(); - logger.info("shard {} count = {}", me.getKey(), - commonShardStats.getIndexing().getTotal().getIndexCount()); - } - } - try { - client.deleteIndex("test1").deleteIndex("test2"); - } catch (Exception e) { - logger.error("delete index failed, ignored. Reason:", e); - } - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - public void testUpdateReplicaLevel() throws Exception { - - long numberOfShards = 2; - int replicaLevel = 3; - - // we need 3 nodes for replica level 3 - startNode("2"); - startNode("3"); - - int shardsAfterReplica; - - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) - .build(); - - Settings settings = Settings.settingsBuilder() - .put("index.number_of_shards", numberOfShards) - .put("index.number_of_replicas", 0) - .build(); - - try { - client.newIndex("replicatest", settings, new HashMap<>()); - client.waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - for (int i = 0; i < 12345; i++) { - client.index("replicatest", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.updateReplicaLevel("replicatest", replicaLevel, 30L, TimeUnit.SECONDS); - assertEquals(replicaLevel, client.getReplicaLevel("replicatest")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - } - } -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java b/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java deleted file mode 100644 index 7abcc5a..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package org.xbib.elx.transport; \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java similarity index 67% rename from elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java index c4dc4fa..54b68c5 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/ClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.transport; +package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -13,69 +13,87 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Before; -import org.junit.Test; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import org.xbib.elx.transport.ExtendedTransportClient; +import org.xbib.elx.transport.ExtendedTransportClientProvider; -import java.util.HashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class ClientTest extends TestBase { +@ExtendWith(TestExtension.class) +class ClientTest { private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); + private static final Long ACTIONS = 100L; + private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; - private static final Long ACTIONS = 1234L; + private final TestExtension.Helper helper; - @Before - public void startNodes() { - try { - super.startNodes(); - startNode("2"); - } catch (Throwable t) { - logger.error("startNodes failed", t); - } + ClientTest(TestExtension.Helper helper) { + this.helper = helper; + helper.startNode("2"); } @Test - public void testClientIndexOp() throws Exception { + void testClientIndexOp() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); - client.newIndex("test"); - try { - client.deleteIndex("test") - .newIndex("test") - .deleteIndex("test"); - } catch (NoNodeAvailableException e) { - logger.error("no node available"); - } finally { - client.close(); - } + client.newIndex("test1"); + client.close(); } @Test - public void testSingleDoc() throws Exception { + void testMapping() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + client.newIndex("test2", Settings.EMPTY, builder.string()); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2"); + GetMappingsResponse getMappingsResponse = + client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); + logger.info("mappings={}", getMappingsResponse.getMappings()); + assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc")); + client.close(); + } + + @Test + void testSingleDoc() throws Exception { + final ExtendedTransportClient client = ClientBuilder.builder() + .provider(ExtendedTransportClientProvider.class) + .put(helper.getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); + client.newIndex("test3"); + client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}"); client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { @@ -91,47 +109,22 @@ public class ClientTest extends TestBase { } @Test - public void testMapping() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - XContentBuilder builder = jsonBuilder() - .startObject() - .startObject("doc") - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject() - .endObject(); - client.newIndex("test", Settings.EMPTY, builder.string()); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); - client.close(); - } - - @Test - public void testRandomDocs() throws Exception { + void testRandomDocs() throws Exception { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test"); + client.newIndex("test4"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test4", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + client.waitForResponses(60L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { @@ -140,37 +133,40 @@ public class ClientTest extends TestBase { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); + client.refreshIndex("test4"); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) + .setIndices("test4") + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0); + assertEquals(numactions, + searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.close(); } } @Test - public void testThreadedRandomDocs() throws Exception { + @Disabled + void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); long maxactions = MAX_ACTIONS_PER_REQUEST; final long maxloop = ACTIONS; - - Settings settingsForIndex = Settings.settingsBuilder() - .put("index.number_of_shards", 2) - .put("index.number_of_replicas", 1) - .build(); - final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test", settingsForIndex, new HashMap<>()) - .startBulk("test", -1, 1000); - ThreadPoolExecutor pool = EsExecutors.newFixed("bulkclient-test", maxthreads, 30, - EsExecutors.daemonThreadFactory("bulkclient-test")); + client.newIndex("test5") + .startBulk("test5", -1, 1000); + ThreadPoolExecutor pool = EsExecutors.newFixed("transportclient-test", maxthreads, 30, + EsExecutors.daemonThreadFactory("transportclient-test")); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { pool.execute(() -> { for (int i1 = 0; i1 < maxloop; i1++) { - client.index("test",null, false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5",null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); @@ -179,25 +175,25 @@ public class ClientTest extends TestBase { if (latch.await(60, TimeUnit.SECONDS)) { logger.info("flush ..."); client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - logger.info("pool to be shut down ..."); + client.waitForResponses(60L, TimeUnit.SECONDS); + logger.info("pool shutdown ..."); pool.shutdown(); - logger.info("poot shut down"); + logger.info("poot is shut down"); + } else { + logger.warn("latch timeout"); } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { + client.stopBulk("test5", 60L, TimeUnit.SECONDS); + assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); - // extra search lookup - client.refreshIndex("test"); + client.refreshIndex("test5"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - // to avoid NPE at org.elasticsearch.action.search.SearchRequest.writeTo(SearchRequest.java:580) - .setIndices("_all") + .setIndices("test5") .setQuery(QueryBuilders.matchAllQuery()) .setSize(0); assertEquals(maxthreads * maxloop, diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java similarity index 60% rename from elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index 279fb6b..8172f89 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.transport; +package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,47 +7,57 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; +import org.xbib.elx.transport.ExtendedTransportClient; +import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class DuplicateIDTest extends TestBase { +@ExtendWith(TestExtension.class) +class DuplicateIDTest { private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private final static Long MAX_ACTIONS_PER_REQUEST = 1000L; + private final static Long MAX_ACTIONS_PER_REQUEST = 10L; - private final static Long ACTIONS = 12345L; + private final static Long ACTIONS = 5L; + + private final TestExtension.Helper helper; + + DuplicateIDTest(TestExtension.Helper helper) { + this.helper = helper; + } @Test - public void testDuplicateDocIDs() throws Exception { + void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { - client.newIndex("test"); + client.newIndex("test_dup"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test_dup", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test"); + client.refreshIndex("test_dup"); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.types("test"); + searchRequest.indices("test_dup"); searchRequest.source(builder); - long hits = client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); + long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java similarity index 62% rename from elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 4ce1843..de886d1 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.transport; +package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,9 +7,12 @@ import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsReques import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.transport.ExtendedTransportClient; +import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -17,44 +20,51 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexPruneTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexPruneTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexPruneTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testPrune() throws IOException { + void testPrune() throws IOException { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1", settings); - client.shiftIndex("test", "test1", Collections.emptyList()); - client.newIndex("test2", settings); - client.shiftIndex("test", "test2", Collections.emptyList()); - client.newIndex("test3", settings); - client.shiftIndex("test", "test3", Collections.emptyList()); - client.newIndex("test4", settings); - client.shiftIndex("test", "test4", Collections.emptyList()); + client.newIndex("test_prune1", settings); + client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + client.newIndex("test_prune2", settings); + client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + client.newIndex("test_prune3", settings); + client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + client.newIndex("test_prune4", settings); + client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test", "test4", 2, 2, true); + client.pruneIndex("test_prune", "test_prune4", 2, 2, true); - assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); - assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); - assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); - assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); + assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); + assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); + assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); List list = new ArrayList<>(); - for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { + for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); indicesExistsRequest.indices(new String[] { index }); IndicesExistsResponse indicesExistsResponse = diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java similarity index 73% rename from elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 41388c7..1c57312 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.transport; +package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,26 +7,36 @@ import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.transport.ExtendedTransportClient; +import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.util.Arrays; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -public class IndexShiftTest extends TestBase { +@ExtendWith(TestExtension.class) +class IndexShiftTest { private static final Logger logger = LogManager.getLogger(IndexShiftTest.class.getName()); + private final TestExtension.Helper helper; + + IndexShiftTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testIndexAlias() throws Exception { + void testIndexAlias() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()).build(); + .put(helper.getTransportSettings()).build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) @@ -34,13 +44,14 @@ public class IndexShiftTest extends TestBase { .build(); client.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test1234", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); + client.shiftIndex("test_shift", "test1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); @@ -51,23 +62,24 @@ public class IndexShiftTest extends TestBase { assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("test")); + assertTrue(aliases.containsKey("test_shift")); - String resolved = client.resolveAlias("test"); + String resolved = client.resolveAlias("test_shift"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("test")); + assertTrue(aliases.containsKey("test_shift")); client.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", randomString(1), false, "{ \"name\" : \"" + randomString(32) + "\"}"); + client.index("test5678", helper.randomString(1), false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + indexShiftResult = client.shiftIndex("test_shift", "test5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, index, alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -86,7 +98,7 @@ public class IndexShiftTest extends TestBase { assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - resolved = client.resolveAlias("test"); + resolved = client.resolveAlias("test_shift"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); @@ -98,11 +110,11 @@ public class IndexShiftTest extends TestBase { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { + client.close(); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); - client.close(); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java new file mode 100644 index 0000000..4127d22 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/MockNode.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport.test; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.Node; + +public class MockNode extends Node { + + public MockNode(Settings settings) { + super(settings); + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java similarity index 61% rename from elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java rename to elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index 3721157..0572ef5 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -1,46 +1,50 @@ -package org.xbib.elx.transport; +package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; -import org.junit.Test; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.transport.ExtendedTransportClient; +import org.xbib.elx.transport.ExtendedTransportClientProvider; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; -public class SmokeTest extends TestBase { +@ExtendWith(TestExtension.class) +class SmokeTest extends TestExtension { private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); + private final TestExtension.Helper helper; + + SmokeTest(TestExtension.Helper helper) { + this.helper = helper; + } + @Test - public void testSingleDocNodeClient() throws Exception { + void testSingleDocNodeClient() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) - .put(getTransportSettings()) + .put(helper.getTransportSettings()) .build(); try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.newIndex("test_smoke"); + client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - - assertEquals(getClusterName(), client.getClusterName()); - - client.checkMapping("test"); - - client.update("test", "1", "{ \"name\" : \"Another name\"}"); + assertEquals(helper.getCluster(), client.getClusterName()); + client.checkMapping("test_smoke"); + client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); client.flush(); - - client.waitForRecovery("test", 10L, TimeUnit.SECONDS); - - client.delete("test", "1"); - client.deleteIndex("test"); - + client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); + client.delete("test_smoke", "1"); + client.deleteIndex("test_smoke"); IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test2", Settings.settingsBuilder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); @@ -48,7 +52,6 @@ public class SmokeTest extends TestBase { client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.flush(); client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java new file mode 100644 index 0000000..8c6fe19 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java @@ -0,0 +1,229 @@ +package org.xbib.elx.transport.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.support.AbstractClient; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.InetSocketTransportAddress; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.node.Node; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; + +public class TestExtension implements ParameterResolver, BeforeAllCallback, AfterAllCallback { + + private static final Logger logger = LogManager.getLogger("test"); + + private static final Random random = new Random(); + + private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); + + private Map nodes = new HashMap<>(); + + private Map clients = new HashMap<>(); + + private String home; + + private String cluster; + + private String host; + + private int port; + + private static final String key = "es-instance"; + + private static final ExtensionContext.Namespace ns = + ExtensionContext.Namespace.create(TestExtension.class); + + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return parameterContext.getParameter().getType().equals(Helper.class); + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); + } + + @Override + public void beforeAll(ExtensionContext context) throws Exception { + Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); + setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + setClusterName("test-cluster-" + System.getProperty("user.name")); + deleteFiles(Paths.get(getHome() + "/data")); + logger.info("data files wiped: " + getHome()); + Thread.sleep(2000L); // let OS commit changes + logger.info("starting cluster"); + helper.startNode("1"); + NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); + NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); + Object obj = response.iterator().next().getTransport().getAddress() + .publishAddress(); + if (obj instanceof InetSocketTransportAddress) { + InetSocketTransportAddress address = (InetSocketTransportAddress) obj; + host = address.address().getHostName(); + port = address.address().getPort(); + } + try { + ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN) + .timeout(TimeValue.timeValueSeconds(30))).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + throw new IOException("cluster state is " + healthResponse.getStatus().name() + + ", from here on, everything will fail!"); + } + } catch (ElasticsearchTimeoutException e) { + throw new IOException("cluster does not respond to health request, cowardly refusing to continue"); + } + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateResponse clusterStateResponse = + helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); + logger.info("host = {} port = {}", host, port); + } + + @Override + public void afterAll(ExtensionContext context) throws Exception { + closeNodes(); + deleteFiles(Paths.get(getHome() + "/data")); + logger.info("cluster stopped"); + } + + private void setClusterName(String cluster) { + this.cluster = cluster; + } + + private String getClusterName() { + return cluster; + } + + private void setHome(String home) { + this.home = home; + } + + private String getHome() { + return home; + } + + private void closeNodes() { + logger.info("closing all clients"); + for (AbstractClient client : clients.values()) { + client.close(); + } + clients.clear(); + logger.info("closing all nodes"); + for (Node node : nodes.values()) { + if (node != null) { + node.close(); + } + } + nodes.clear(); + logger.info("all nodes closed"); + } + + private static void deleteFiles(Path directory) throws IOException { + if (Files.exists(directory)) { + Files.walkFileTree(directory, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + } + + private Helper create() { + return new Helper(); + } + + class Helper { + + Settings getNodeSettings() { + return settingsBuilder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } + + Settings getTransportSettings() { + return settingsBuilder() + .put("host", host) + .put("port", port) + .put("cluster.name", getClusterName()) + .put("path.home", getHome() + "/transport") + .build(); + } + + void startNode(String id) { + buildNode(id).start(); + } + + private Node buildNode(String id) { + Settings nodeSettings = settingsBuilder() + .put(getNodeSettings()) + .put("name", id) + .build(); + Node node = new MockNode(nodeSettings); + AbstractClient client = (AbstractClient) node.client(); + nodes.put(id, node); + clients.put(id, client); + logger.info("clients={}", clients); + return node; + } + + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String getCluster() { + return getClusterName(); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/package-info.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/package-info.java new file mode 100644 index 0000000..b038c6f --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/package-info.java @@ -0,0 +1 @@ +package org.xbib.elx.transport.test; \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index ae3020f..c904bfb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,18 +1,19 @@ group = org.xbib name = elx -version = 2.2.1.7 +version = 2.2.1.8 xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 +xbib-netty-http.version = 4.1.35.0 elasticsearch.version = 2.2.1 +jackson.version = 2.6.7 jna.version = 4.5.2 log4j.version = 2.11.1 mustache.version = 0.9.5 jts.version = 1.13 -jackson-dataformat.version = 2.8.11 -junit.version = 4.12 +junit.version = 5.4.2 wagon.version = 3.0.0 asciidoclet.version = 1.5.4 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 87b738cbd051603d91cc39de6cb000dd98fe6b02..5c2d1cf016b3885f6930543d57b744ea8c220a1a 100644 GIT binary patch delta 3320 zcmai0c|2768`iN!wwN(!Oxeo5?`tVU3{m#%jC~noTx!q_nHtNnR`zAgWC@krB#b55 znJk4YA);()+(!K-w|npJuix)IpYu7-^SqzuJ>T~|?;j_-ma(;-@!<_I_B>B@4FVej z11CRtM@$8afpkN^v*te{ycR9yTldxXJbmio?@}x{9}zaw&=aQt(a^ZXN9S3i8a+Z% zGc@&(5}jplZjJKk2wNlTp(mbeKL5J9Gjo==yT{-eVKj?*rT1%bQ@%#Xce~~1f{19^ zoD75QEoSzDVh@!9qG4yl`;9=Ysp?rRX=(8$VDRz=R+oA3>jLxjW-H!-2biNSYuy)U z7-B-qC5l;>qjMTg!DbWPY}h7qxi6xp)_T)_O2+*&NDg?v;RyY@5XtWHx%(ImQ_3E% zA%$s3xrxE0Fk>DhG!pG)4}I!pWJl~QtV_3Jl2W4PuWWssMq^UpGatK+4CING9pB#5 z_NDc)aonVrZuXsr5!RcE#?aXFZQjt2VMd)-p00K$EheT?H!m_D2Mdqq;0moaO=C&y zgJnvzgUn!wkx^{r049pU#gsIMhl`%{MDNl;}JRbneC zSTB=5f;o9=2Rt24_lt&%%f~m{Ts)zu8H9j`INrgMp>l-|k%Kj%U`OXL1J2e+CJHJxreHLD_#o*ZeuXE4uGDQAJS_PpEGt7hmd7psmLEBL^h zD#JbHiklZEXkk9(6uF$ErsUu^jg7c~1oRS&CuTq*Xg_cOvGw~FZ&1#p(6|jz9lJnP zSIJ)sX_W2$PSksX&}*_ejz+t*X)xK|JcakaMRGd%c*R)cQcT|?sM^#{fdjh5_I$iK zBX_d;wz+cf>b}r!i3yo6eaua)d`|Mi_|Q3mAz5Qn?#~xgE9In<;TwYN^~mtaYy#WU z*ffWtxwlk&!e@UfqQ$bn23RDFV3o-H_WM}44yQpYw;JuRf$at#XX-qmuVnKqg-Bo# zJjZE39)!{i$qJh?oJzVzWFDlSW;{Wf`Z)33Y$Fh^+qasrsEJsfy9yhyTFe?Lej&3n zEAS(D8WCt(ew(SGD z-J#7@l?KI*ZbS)AVQ23qV&{c=$@zUp0@6=kZp+5by+gnAWdB||7e=!yJ|WTpG0OC7 zKlKWFv6#(>nrEq@d1i-#L9SVxTDNb1DaY%2$=@)`k&3s8wz$M*;THa&!2Isj%6CQS zY>A4HtmWY3@9e@F)mCHJQzBz~Lt(wcJE{!CAr=wxn4|5n(jslTy)~IF?tNK zD^2#hTM0d6MDg>`9;s5*(4W1V8y}F8OT6Xap{`=h1XVKO3zrBh=;JnIs*RB>@7t5T zwV=G^T)L=(9P7tS={6`tEBBBm^u~_!-#m75G*h}y_Jj7|STtiY_LDR5UUHI@awWmB zDn6q9{2M-EHaTm53ln%ENJ$HpLwRcL>7^hUrM=}&`qmWTgtr{Ul*Lqcd_9S0xZ1s>F2dVd(s)3&$`gxFAu6jXYIS ze#M~w@=X@lm)sFI4EEiqKh7JxN=_?+}D=iHCc&S2<^VPZ6 zYKXZgvi(Yne9}k6o=ezgquABVB77}x$nKXh`@LjH&lQPqm_;MTL>4RGO|E#_7AS4@43rz=ij?gcMZalnd-JK4ILhL)Ee(3G zN}g99HmhxoBjHR~y@b>-7{f+`p zIZ<^8%d;wCA#xfwSc6$DNVPjAX6FCkb|MQ|6hFyz9UhoLF0^xUd#*^2Ofn zOJgmwDyb1=Z8T)ArRy|VQOM+BrhZ>W_ELJ6u(d^JTu|j%*6g8JKZ-ewoj)sXJCdS= zHOo?HscL;Z`H18}%WnE1&o42KZ+=fg(*VN>t>kRkcd{mP9NF6;MnzH&m2WsD)sX~h zbhv|Ux$w2avQwoI`IKiGMLrL;Z>R}Y_0K*L=63V z)ut+5tM74Glzb?92kbu5@3M#1Hi7K3$c)?TL$}`aKf0hC3`r!>Xy3!f{ z`}Y#@$`|mG1JlKzVE!vD04aX}x#hV*+AC>bQ|%XJ1<&;=0?uX!RM?CIB=+!tgkB-w zu*HF--^U4#nG1mXz0v^0@|UCs1lt}!1zTaTwoe+k?sPym`pyB-F25ivXx)#1|1%|e zJ7Vpujkk#Lu%U{v6xiQ5LW2`~QXrR`ja@*L=b0ejT977v%C)0WAik0gV7U z6a-7##p#p>>>3a{^Z}e3Z~?A|foBFU12bqaEE*0vqdCCVLFq%{;F%$Dkb6i8;Qo!C z&;zkU(!i5zbSMd)zQzg8(kU^HPQ^flVIzR)<^jwbwget09YD?zV*rx+mx@0IN{#S< zsB|8Ve>>sJI7sHE!@=(((ttqL0ks%C4M^r5!0H?rJ;MV|jtT)1cMl{|9xo_Okp@Ka ze^CzbCPf?IDFWLlE`V1FDDpZ0C@7~VMZt%!6%SFtxz{!Tb1UfBDEg~49x!4|2#_L! zX=6UXeh28_?VY*suC^Sy!?XXp?9-G{ zEbF`ELqycMcTK-$-pw|Jox9S^<_NX$7{PI7aX1p5N>aOyj&D01H#;3?=q^!=_mq@k zUHheWO_|CDYA~8r<-%q8&Gm$uPSx4S`reKPnv?Nif4kS)^smTg&m@kLYT87txGxGxw+Qc zTAi=`vzavOlyLrgf2A~;1~Gx$jcb|fkhfctRt6CjRooL|#wr)(*8D4n;2cBe>p9_T zCeJf!IgCH0h1m)UPLk3hZz120oe5YH$oXjSMHcPv@#wX;OP5bBSJMavm2}5Q8(V&# zXGA!+dAwOiXuQ)|+XwF2HW1@_MPm3*v{M86V_~+xk1K7cI7mxBKU5#bofCjZqqjs$ z(sipv#Ul%KJ)h?ua}a3Dg(6yaxeJ(HD-&`AT9kZJVLJTz?WIfgao$bYwEhXh+&GA= zkpI03HVxtWc*H!~z~9%DC;;Qej=WppOD!i1$MO1`&8LW%IWd2sbnS7j+<0b`v1%qx!owUU+ZIHJFp1yH9BFvUYI^up=ZYX$K_YM|Bn2fCG3sq#(EpRB$|A9~9*^M%Sq)EAjr0&W`hHyz96Z9h*odHK|Ju$JQ0c zO9oayZQv;2b{pLJo`T)C%yS@sAKO*WC%22XDmrdRTd;uFr*sb_{GDl=*Y`l*;>lNWh=XCbn#V}C&jmw3>t zNH(fnG%j@AI$TSggf(e3DxrpHjnpeKExsb|hC`kxjD4HUSmu)&aJNt&DtCWh#51*} zS!qfplP(f0`hJ)VHrXFD_uB7ia4#%U)3S8lGY9^(T1)M8xQxP*3w4&QJr~O`$A&N5 z_taom$34zt+reJDV?oZ*qr5ERUH7#~xm7)D(u#q#m`~~-F+TZ6Q*L)s_#T3GZUuZM zhCH9!{qXnD)9jln$|GDeDPqo=+D6#vQkAjdHtT>{VxU#AQJW-je=UWN5*R>v5vWF6 zK_6z?#thq>&%@fu5epvO$rfx`v9GojdOLGFaQ2V8?Ri z(?L2JBK(;G)bIF7r5T6Ahzst5k4j#hvhl3a`@Ksfyj3^Cx}zGE)vm$ecB$?~2`S&e zE)Nx6TiDO*JO6UmWWc+zLDmnII+)ROEvW3_{*%Fjs8Q^k4+Z&cJ0lp=@p*N!fw0>L zPSWrxar=HPDCwZnmN%orA-K2142{bJ0el>N{KM(xoHJu_HWSQihq^y%SEmj>CsBjl zj6)jxqm7NwiVHh-xQ`ex^02-y_ZO`A`P(1UwLK5G_T8=uI8@e%Kh31Xay z>H$7OG8cQ%>c_RjXhRA|Yh=93MnM)V0JlD#yP-1YNx}5`sg}-vE%slfve&}e$*L>+ zSAq_CMc5SYx6N)5h%-)?JOAhiVM5`TWT7?<9 zKKxMMb9GXHpQ1ajAr?!hxcauobJLf{IpvJ=9ny}FwdGCYmwgj?0qhIG{5zbTTVc2b zo+3h|{F_Yg96k{?rVn`m`%d??#avI-eh^XnTH2r*o>5n>`UuIsuCIeN5Br62W!Yy#8)0uWcVG%-QnMHczpWoe zftoSf-WJq~x8`|ws<-9{Va9@s#SoH3uw`>4!~uyB-(lV)SD9f(TPNa!o7JLL%!a)@gUmedno%~}$ z#zZLYah$5mf@Z2}a(oDDM^$qq>*nb;?aVn?D`($Om=?j+T%S?eSgR1t=zzwGw|kvM zt~WiOO&UVW=7N=8ERxM<4?Wbj4bPIP4z3=hjp(uuT}ne*E9ct0)Lsk?bG=1nNo=oB z0JEoKzAw45q-lB!IbJKsY=Lpru48qY6ql!Z#J13ywC&7??l&AtxiowZ|Cg(k*UE#@ zrJm|m^EV_6jz}f($PrOb`S;imdEwtu`#cCu3aMXBgUUH4t2j_qu=KmOO645(v(_DL z^G5PF%RR0@X5D{(V%x5L{xD1Sa>^wR+$0j(DeVfwk;tp3<@i$~qOsvx^uUy!zV8G0~0`$f?VV=?vm zOwYnZB>UV_b#sh6ibtN`5I+l%mTE9T%*J!xaz}cWisUNLg@>nEiKv4hgmv`5C)GIDbBOgq{?5K-!=>z{CLJ$wIBkL-~yV{}~e*^#eZ1f%)RR;DgcM zfOqnA#42!t$D;@!QT3n50ve1d0$Zl^m}ABc){bz2HDhq#o&{ZLlQ=*lO9Alv7y_uW z`bTL2KkVsP<{%6$`1yeL}DmCZuxPZRJp*( z*Kk1M23@g@UjhQ6PEZ{58CL@Aqv>cB0|#ltT;SR`95{}ptMe0@zz&v<>j{GNDt-bE zn5EFw?u0e)Ee+J0^aq@C>E_j>A%MyU^@?Rcohe{^TCd{d<=ub5$bWAh Date: Thu, 2 May 2019 21:34:49 +0200 Subject: [PATCH 08/19] semaphore logging --- build.gradle | 1 + .../java/org/xbib/elx/common/DefaultBulkProcessor.java | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/build.gradle b/build.gradle index 1333813..df2dfa7 100644 --- a/build.gradle +++ b/build.gradle @@ -65,6 +65,7 @@ subprojects { } test { + enabled = false useJUnitPlatform() // we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery doFirst { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index e8048cc..c633ad1 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -13,12 +13,15 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.FutureUtils; import org.xbib.elx.api.BulkProcessor; +import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import java.util.logging.Logger; /** * A bulk processor is a thread safe bulk processing class, allowing to easily set when to "flush" a new bulk request @@ -28,6 +31,8 @@ import java.util.concurrent.atomic.AtomicLong; */ public class DefaultBulkProcessor implements BulkProcessor { + private static final Logger logger = Logger.getLogger(DefaultBulkProcessor.class.getName()); + private final int bulkActions; private final long bulkSize; @@ -421,6 +426,10 @@ public class DefaultBulkProcessor implements BulkProcessor { @Override public boolean close(long timeout, TimeUnit unit) throws InterruptedException { + logger.log(Level.INFO, "semaphore=" + semaphore + + " concurrentRequests=" + concurrentRequests + + " timeout=" + timeout + + " unit=" + unit); if (semaphore.tryAcquire(concurrentRequests, timeout, unit)) { semaphore.release(concurrentRequests); return true; From 43c983f2a92f5c85100646ba5997ffd587fd30aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 2 May 2019 21:37:55 +0200 Subject: [PATCH 09/19] update 2.2.1.9 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index c904bfb..7e22f6c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.8 +version = 2.2.1.9 xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 From 979b06fbf2c10d530dcca3c78c95976f363c724a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Fri, 3 May 2019 11:31:00 +0200 Subject: [PATCH 10/19] enable tests, add NPE guards to DefaultBulkProcessor --- build.gradle | 2 +- .../xbib/elx/common/DefaultBulkProcessor.java | 33 ++++------ .../xbib/elx/common/test/TestExtension.java | 50 ++++++++------- .../org/xbib/elx/node/test/SmokeTest.java | 2 +- .../org/xbib/elx/node/test/TestExtension.java | 57 +++++++++-------- .../xbib/elx/transport/test/ClientTest.java | 3 - .../elx/transport/test/DuplicateIDTest.java | 4 +- .../elx/transport/test/IndexShiftTest.java | 2 +- .../xbib/elx/transport/test/SmokeTest.java | 2 +- .../elx/transport/test/TestExtension.java | 64 ++++++++++--------- 10 files changed, 107 insertions(+), 112 deletions(-) diff --git a/build.gradle b/build.gradle index df2dfa7..5273aa6 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ subprojects { } test { - enabled = false + enabled = true useJUnitPlatform() // we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery doFirst { diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index c633ad1..74915c9 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -20,8 +20,6 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.logging.Level; -import java.util.logging.Logger; /** * A bulk processor is a thread safe bulk processing class, allowing to easily set when to "flush" a new bulk request @@ -31,8 +29,6 @@ import java.util.logging.Logger; */ public class DefaultBulkProcessor implements BulkProcessor { - private static final Logger logger = Logger.getLogger(DefaultBulkProcessor.class.getName()); - private final int bulkActions; private final long bulkSize; @@ -56,9 +52,6 @@ public class DefaultBulkProcessor implements BulkProcessor { this.bulkActions = bulkActions; this.bulkSize = bulkSize.getBytes(); this.bulkRequest = new BulkRequest(); - if (listener == null) { - throw new IllegalArgumentException(); - } this.bulkRequestHandler = concurrentRequests == 0 ? new SyncBulkRequestHandler(client, listener) : new AsyncBulkRequestHandler(client, listener, concurrentRequests); @@ -76,9 +69,8 @@ public class DefaultBulkProcessor implements BulkProcessor { } public static Builder builder(ElasticsearchClient client, Listener listener) { - if (client == null) { - throw new NullPointerException("The client you specified while building a BulkProcessor is null"); - } + Objects.requireNonNull(client, "The client you specified while building a BulkProcessor is null"); + Objects.requireNonNull(listener, "A listener for the BulkProcessor is required but null"); return new Builder(client, listener); } @@ -91,6 +83,7 @@ public class DefaultBulkProcessor implements BulkProcessor { */ @Override public synchronized boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException { + Objects.requireNonNull(unit, "A time unit is required for awaitFlush() but null"); if (closed) { return true; } @@ -99,7 +92,7 @@ public class DefaultBulkProcessor implements BulkProcessor { execute(); } // wait for all bulk responses - return this.bulkRequestHandler.close(timeout, unit); + return bulkRequestHandler.close(timeout, unit); } /** @@ -119,18 +112,19 @@ public class DefaultBulkProcessor implements BulkProcessor { */ @Override public synchronized boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException { + Objects.requireNonNull(unit, "A time unit is required for awaitCLose() but null"); if (closed) { return true; } closed = true; - if (this.scheduledFuture != null) { - FutureUtils.cancel(this.scheduledFuture); - this.scheduler.shutdown(); + if (scheduledFuture != null) { + FutureUtils.cancel(scheduledFuture); + scheduler.shutdown(); } if (bulkRequest.numberOfActions() > 0) { execute(); } - return this.bulkRequestHandler.close(timeout, unit); + return bulkRequestHandler.close(timeout, unit); } /** @@ -213,8 +207,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private boolean isOverTheLimit() { return bulkActions != -1 && bulkRequest.numberOfActions() >= bulkActions || - bulkSize != -1 && - bulkRequest.estimatedSizeInBytes() >= bulkSize; + bulkSize != -1 && bulkRequest.estimatedSizeInBytes() >= bulkSize; } /** @@ -342,6 +335,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private final DefaultBulkProcessor.Listener listener; SyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener) { + Objects.requireNonNull(listener, "A listener is required for SyncBulkRequestHandler but null"); this.client = client; this.listener = listener; } @@ -378,6 +372,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private final int concurrentRequests; private AsyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { + Objects.requireNonNull(listener, "A listener is required for AsyncBulkRequestHandler but null"); this.client = client; this.listener = listener; this.concurrentRequests = concurrentRequests; @@ -426,10 +421,6 @@ public class DefaultBulkProcessor implements BulkProcessor { @Override public boolean close(long timeout, TimeUnit unit) throws InterruptedException { - logger.log(Level.INFO, "semaphore=" + semaphore + - " concurrentRequests=" + concurrentRequests + - " timeout=" + timeout + - " unit=" + unit); if (semaphore.tryAcquire(concurrentRequests, timeout, unit)) { semaphore.release(concurrentRequests); return true; diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java index fdf65e6..6186f8c 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java @@ -19,8 +19,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -39,7 +39,7 @@ import java.util.Random; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -public class TestExtension implements ParameterResolver, BeforeAllCallback, AfterAllCallback { +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { private static final Logger logger = LogManager.getLogger("test"); @@ -73,18 +73,15 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + setHome(System.getProperty("path.home") + "/" + getRandomString(8)); + setClusterName("test-cluster-" + System.getProperty("user.name")); return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); } @Override - public void beforeAll(ExtensionContext context) throws Exception { + public void beforeEach(ExtensionContext context) throws Exception { Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); - setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); - setClusterName("test-cluster-" + System.getProperty("user.name")); logger.info("starting cluster"); - deleteFiles(Paths.get(getHome() + "/data")); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes helper.startNode("1"); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); @@ -114,9 +111,11 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte } @Override - public void afterAll(ExtensionContext context) throws Exception { + public void afterEach(ExtensionContext context) throws Exception { closeNodes(); deleteFiles(Paths.get(getHome() + "/data")); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes } private void setClusterName(String cluster) { @@ -169,6 +168,15 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte } } + private String getRandomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + private Helper create() { return new Helper(); } @@ -186,6 +194,14 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte buildNode(id).start(); } + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String randomString(int n) { + return getRandomString(n); + } + private Node buildNode(String id) { Settings nodeSettings = settingsBuilder() .put(getNodeSettings()) @@ -198,19 +214,5 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte logger.info("clients={}", clients); return node; } - - String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - ElasticsearchClient client(String id) { - return clients.get(id); - } - } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index ae33bd9..c9a7cd7 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -44,7 +44,7 @@ class SmokeTest { client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); client.delete("test_smoke", "1"); client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.settingsBuilder() + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke_2", Settings.settingsBuilder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); client.newIndex(indexDefinition); diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java index 7d28686..dcd0e31 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java @@ -19,8 +19,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -39,7 +39,7 @@ import java.util.Random; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -public class TestExtension implements ParameterResolver, BeforeAllCallback, AfterAllCallback { +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { private static final Logger logger = LogManager.getLogger("test"); @@ -73,17 +73,14 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { + setHome(System.getProperty("path.home") + "/" + getRandomString(8)); + setClusterName("test-cluster-" + System.getProperty("user.name")); return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); } @Override - public void beforeAll(ExtensionContext context) throws Exception { + public void beforeEach(ExtensionContext context) throws Exception { Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); - setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); - setClusterName("test-cluster-" + System.getProperty("user.name")); - deleteFiles(Paths.get(getHome() + "/data")); - logger.info("data files wiped"); - Thread.sleep(2000L); // let OS commit changes logger.info("starting cluster"); helper.startNode("1"); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); @@ -114,9 +111,11 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte } @Override - public void afterAll(ExtensionContext context) throws Exception { + public void afterEach(ExtensionContext context) throws Exception { closeNodes(); deleteFiles(Paths.get(getHome() + "/data")); + logger.info("data files wiped"); + Thread.sleep(2000L); // let OS commit changes } private void setClusterName(String cluster) { @@ -169,6 +168,15 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte } } + private String getRandomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + private Helper create() { return new Helper(); } @@ -179,6 +187,18 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte buildNode(id).start(); } + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String getCluster() { + return getClusterName(); + } + + String randomString(int n) { + return getRandomString(n); + } + private Node buildNode(String id) { Settings nodeSettings = settingsBuilder() .put("cluster.name", getClusterName()) @@ -192,22 +212,5 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte logger.info("clients={}", clients); return node; } - - String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - ElasticsearchClient client(String id) { - return clients.get(id); - } - - String getCluster() { - return getClusterName(); - } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java index 54b68c5..73c0f81 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java @@ -13,7 +13,6 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; @@ -43,7 +42,6 @@ class ClientTest { ClientTest(TestExtension.Helper helper) { this.helper = helper; - helper.startNode("2"); } @Test @@ -145,7 +143,6 @@ class ClientTest { } @Test - @Disabled void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); long maxactions = MAX_ACTIONS_PER_REQUEST; diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index 8172f89..64207e7 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -25,9 +25,9 @@ class DuplicateIDTest { private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private final static Long MAX_ACTIONS_PER_REQUEST = 10L; + private final static Long MAX_ACTIONS_PER_REQUEST = 100L; - private final static Long ACTIONS = 5L; + private final static Long ACTIONS = 50L; private final TestExtension.Helper helper; diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 1c57312..1f65064 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -33,7 +33,7 @@ class IndexShiftTest { } @Test - void testIndexAlias() throws Exception { + void testIndexShift() throws Exception { final ExtendedTransportClient client = ClientBuilder.builder() .provider(ExtendedTransportClientProvider.class) .put(helper.getTransportSettings()).build(); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index 0572ef5..be4a87b 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -45,7 +45,7 @@ class SmokeTest extends TestExtension { client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); client.delete("test_smoke", "1"); client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test2", Settings.settingsBuilder() + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke_2", Settings.settingsBuilder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); client.newIndex(indexDefinition); diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java index 8c6fe19..e37ca8c 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java @@ -19,8 +19,8 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.InetSocketTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.node.Node; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.ParameterContext; import org.junit.jupiter.api.extension.ParameterResolutionException; @@ -39,7 +39,7 @@ import java.util.Random; import static org.elasticsearch.common.settings.Settings.settingsBuilder; -public class TestExtension implements ParameterResolver, BeforeAllCallback, AfterAllCallback { +public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { private static final Logger logger = LogManager.getLogger("test"); @@ -59,7 +59,7 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte private int port; - private static final String key = "es-instance"; + private static final String key = "es-test-instance"; private static final ExtensionContext.Namespace ns = ExtensionContext.Namespace.create(TestExtension.class); @@ -73,17 +73,14 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); + setHome(System.getProperty("path.home") + "/" + getRandomString(8)); + setClusterName("test-cluster-" + System.getProperty("user.name")); + return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); } @Override - public void beforeAll(ExtensionContext context) throws Exception { - Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); - setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); - setClusterName("test-cluster-" + System.getProperty("user.name")); - deleteFiles(Paths.get(getHome() + "/data")); - logger.info("data files wiped: " + getHome()); - Thread.sleep(2000L); // let OS commit changes + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); logger.info("starting cluster"); helper.startNode("1"); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); @@ -114,10 +111,11 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte } @Override - public void afterAll(ExtensionContext context) throws Exception { + public void afterEach(ExtensionContext context) throws Exception { closeNodes(); deleteFiles(Paths.get(getHome() + "/data")); - logger.info("cluster stopped"); + logger.info("data files wiped: " + getHome()); + Thread.sleep(2000L); // let OS commit changes } private void setClusterName(String cluster) { @@ -170,6 +168,15 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte } } + private String getRandomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); + } + private Helper create() { return new Helper(); } @@ -196,6 +203,18 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte buildNode(id).start(); } + ElasticsearchClient client(String id) { + return clients.get(id); + } + + String getCluster() { + return getClusterName(); + } + + String randomString(int n) { + return getRandomString(n); + } + private Node buildNode(String id) { Settings nodeSettings = settingsBuilder() .put(getNodeSettings()) @@ -208,22 +227,5 @@ public class TestExtension implements ParameterResolver, BeforeAllCallback, Afte logger.info("clients={}", clients); return node; } - - String randomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - - ElasticsearchClient client(String id) { - return clients.get(id); - } - - String getCluster() { - return getClusterName(); - } } } From 7338f4bce1fdf8a9b6d34c234e08c4f91eee1d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 13 Jun 2019 20:22:25 +0200 Subject: [PATCH 11/19] modify for gradle config --- build.gradle | 8 +------- gradle/publish.gradle | 16 ---------------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/build.gradle b/build.gradle index 5273aa6..800b799 100644 --- a/build.gradle +++ b/build.gradle @@ -109,13 +109,7 @@ subprojects { artifacts { archives javadocJar, sourcesJar } - - if (project.hasProperty('signing.keyId')) { - signing { - sign configurations.archives - } - } - + apply from: "${rootProject.projectDir}/gradle/publish.gradle" spotbugs { diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 8675487..4b652e4 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -5,22 +5,6 @@ ext { scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' } -task xbibUpload(type: Upload, dependsOn: build) { - group = 'publish' - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty("xbibUsername")) { - mavenDeployer { - configuration = configurations.wagon - repository(url: uri(project.property('xbibUrl'))) { - authentication(userName: xbibUsername, privateKey: xbibPrivateKey) - } - } - } - } -} - task sonaTypeUpload(type: Upload, dependsOn: build) { group = 'publish' configuration = configurations.archives From d71c164174a4cecc7c01eb3a13f2ed7ce385c353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Tue, 23 Jul 2019 12:52:10 +0200 Subject: [PATCH 12/19] disable bulk logging bey default, update to Gradle 5.3.1 --- build.gradle | 82 +++++++++++++++---- .../java/org/xbib/elx/api/BulkProcessor.java | 2 +- .../elx/common/DefaultBulkController.java | 25 +++--- .../xbib/elx/common/MockExtendedClient.java | 1 - .../java/org/xbib/elx/common/Parameters.java | 12 +++ gradle.properties | 8 +- gradle/publish.gradle | 62 -------------- gradle/wrapper/gradle-wrapper.properties | 4 +- 8 files changed, 99 insertions(+), 97 deletions(-) diff --git a/build.gradle b/build.gradle index 800b799..292a912 100644 --- a/build.gradle +++ b/build.gradle @@ -5,20 +5,6 @@ plugins { id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" } -printf "Host: %s\nOS: %s %s %s\nJVM: %s %s %s %s\nGradle: %s Groovy: %s Java: %s\n" + - "Build: group: ${project.group} name: ${project.name} version: ${project.version}\n", - InetAddress.getLocalHost(), - System.getProperty("os.name"), - System.getProperty("os.arch"), - System.getProperty("os.version"), - System.getProperty("java.version"), - System.getProperty("java.vm.version"), - System.getProperty("java.vm.vendor"), - System.getProperty("java.vm.name"), - gradle.gradleVersion, - GroovySystem.getVersion(), - JavaVersion.current() - if (JavaVersion.current() < JavaVersion.VERSION_11) { throw new GradleException("The build must be run with Java 11") } @@ -34,7 +20,6 @@ subprojects { configurations { asciidoclet - wagon } dependencies { @@ -44,7 +29,6 @@ subprojects { testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}" testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" - wagon "org.apache.maven.wagon:wagon-ssh:${project.property('wagon.version')}" } compileJava { @@ -110,8 +94,6 @@ subprojects { archives javadocJar, sourcesJar } - apply from: "${rootProject.projectDir}/gradle/publish.gradle" - spotbugs { effort = "max" reportLevel = "low" @@ -162,4 +144,68 @@ subprojects { property "sonar.junit.reportsPath", "build/test-results/test/" } } + + ext { + description = 'Extensions for Elasticsearch clients (node and transport)' + scmUrl = 'https://github.com/jprante/elx' + scmConnection = 'scm:git:git://github.com/jprante/elx.git' + scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' + } + + task sonaTypeUpload(type: Upload, dependsOn: build) { + group = 'publish' + configuration = configurations.archives + uploadDescriptor = true + repositories { + if (project.hasProperty('ossrhUsername')) { + mavenDeployer { + beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } + repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { + authentication(userName: ossrhUsername, password: ossrhPassword) + } + pom.project { + groupId project.group + artifactId project.name + version project.version + name project.name + description description + packaging 'jar' + inceptionYear '2019' + url scmUrl + organization { + name 'xbib' + url 'http://xbib.org' + } + developers { + developer { + id 'xbib' + name 'Jörg Prante' + email 'joergprante@gmail.com' + url 'https://github.com/jprante' + } + } + scm { + url scmUrl + connection scmConnection + developerConnection scmDeveloperConnection + } + licenses { + license { + name 'The Apache License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + } + } + } + } + } + } + } + + nexusStaging { + packageGroup = "org.xbib" + } + } \ No newline at end of file diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java index 52b6d04..cb994e0 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -31,7 +31,7 @@ public interface BulkProcessor extends Closeable, Flushable { /** * A listener for the execution. */ - public interface Listener { + interface Listener { /** * Callback before the bulk is executed. diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index f6af929..dbd7fa6 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -49,6 +49,8 @@ public class DefaultBulkController implements BulkController { private AtomicBoolean active; + private boolean enableBulkLogging; + public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) { this.client = client; this.bulkMetric = bulkMetric; @@ -76,11 +78,8 @@ public class DefaultBulkController implements BulkController { ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), "maxVolumePerRequest")); - if (logger.isInfoEnabled()) { - logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + - "flushIngestInterval = {} maxVolumePerRequest = {}", - maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest); - } + this.enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), + Parameters.ENABLE_BULK_LOGGING.getValue()); this.bulkListener = new BulkListener(); this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) @@ -89,6 +88,12 @@ public class DefaultBulkController implements BulkController { .setBulkSize(maxVolumePerRequest) .build(); this.active.set(true); + if (logger.isInfoEnabled()) { + logger.info("bulk processor set up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + + "flushIngestInterval = {} maxVolumePerRequest = {}, bulk logging = {}", + maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest, + enableBulkLogging); + } } @Override @@ -240,7 +245,7 @@ public class DefaultBulkController implements BulkController { private class BulkListener implements DefaultBulkProcessor.Listener { - private final Logger logger = LogManager.getLogger("org.xbib.elx.BulkProcessor.Listener"); + private final Logger logger = LogManager.getLogger(BulkListener.class.getName()); private Throwable lastBulkError = null; @@ -255,7 +260,7 @@ public class DefaultBulkController implements BulkController { bulkMetric.getCurrentIngestNumDocs().inc(n); bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); } - if (logger.isDebugEnabled()) { + if (enableBulkLogging && logger.isDebugEnabled()) { logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", executionId, request.numberOfActions(), @@ -285,7 +290,7 @@ public class DefaultBulkController implements BulkController { } } } - if (bulkMetric != null && logger.isDebugEnabled()) { + if (enableBulkLogging && bulkMetric != null && logger.isDebugEnabled()) { logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", executionId, bulkMetric.getSucceeded().getCount(), @@ -294,7 +299,7 @@ public class DefaultBulkController implements BulkController { l); } if (n > 0) { - if (logger.isErrorEnabled()) { + if (enableBulkLogging && logger.isErrorEnabled()) { logger.error("bulk [{}] failed with {} failed items, failure message = {}", executionId, n, response.buildFailureMessage()); } @@ -312,7 +317,7 @@ public class DefaultBulkController implements BulkController { } lastBulkError = failure; active.set(false); - if (logger.isErrorEnabled()) { + if (enableBulkLogging && logger.isErrorEnabled()) { logger.error("after bulk [" + executionId + "] error", failure); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java index 5cc6cdd..4f91c14 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -6,7 +6,6 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; -import java.io.IOException; import java.util.concurrent.TimeUnit; /** diff --git a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java index 28d10d7..73819e1 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/Parameters.java +++ b/elx-common/src/main/java/org/xbib/elx/common/Parameters.java @@ -2,6 +2,8 @@ package org.xbib.elx.common; public enum Parameters { + ENABLE_BULK_LOGGING(false), + DEFAULT_MAX_ACTIONS_PER_REQUEST(1000), DEFAULT_MAX_CONCURRENT_REQUESTS(Runtime.getRuntime().availableProcessors()), @@ -18,10 +20,16 @@ public enum Parameters { FLUSH_INTERVAL("flush_interval"); + boolean flag; + int num; String string; + Parameters(boolean flag) { + this.flag = flag; + } + Parameters(int num) { this.num = num; } @@ -30,6 +38,10 @@ public enum Parameters { this.string = string; } + boolean getValue() { + return flag; + } + int getNum() { return num; } diff --git a/gradle.properties b/gradle.properties index 7e22f6c..f941f54 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,10 +1,10 @@ group = org.xbib name = elx -version = 2.2.1.9 +version = 2.2.1.10 xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 -xbib-netty-http.version = 4.1.35.0 +xbib-netty-http.version = 4.1.36.7 elasticsearch.version = 2.2.1 jackson.version = 2.6.7 @@ -13,8 +13,10 @@ log4j.version = 2.11.1 mustache.version = 0.9.5 jts.version = 1.13 +# test junit.version = 5.4.2 -wagon.version = 3.0.0 + +# docs asciidoclet.version = 1.5.4 org.gradle.warning.mode = all diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 4b652e4..e69de29 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -1,62 +0,0 @@ -ext { - description = 'Extensions for Elasticsearch clients (node and transport)' - scmUrl = 'https://github.com/jprante/elx' - scmConnection = 'scm:git:git://github.com/jprante/elx.git' - scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' -} - -task sonaTypeUpload(type: Upload, dependsOn: build) { - group = 'publish' - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty('ossrhUsername')) { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - pom.project { - groupId project.group - artifactId project.name - version project.version - name project.name - description description - packaging 'jar' - inceptionYear '2019' - url scmUrl - organization { - name 'xbib' - url 'http://xbib.org' - } - developers { - developer { - id 'xbib' - name 'Jörg Prante' - email 'joergprante@gmail.com' - url 'https://github.com/jprante' - } - } - scm { - url scmUrl - connection scmConnection - developerConnection scmDeveloperConnection - } - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - } - } - } - } -} - -nexusStaging { - packageGroup = "org.xbib" -} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index c5cbb11..f3a4f28 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Thu May 02 09:39:03 CEST 2019 +#Tue Jul 23 11:33:24 CEST 2019 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip From bd59fdffbd3381a69ed892c9b348c207dcadbc89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 8 Aug 2019 22:21:08 +0200 Subject: [PATCH 13/19] update dependencies to current versions --- build.gradle | 8 ++++---- elx-api/build.gradle | 2 ++ .../xbib/elx/common/AbstractExtendedClient.java | 2 +- gradle.properties | 14 ++++++++------ 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 292a912..635759c 100644 --- a/build.gradle +++ b/build.gradle @@ -32,13 +32,13 @@ subprojects { } compileJava { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 } compileTestJava { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 } tasks.withType(JavaCompile) { diff --git a/elx-api/build.gradle b/elx-api/build.gradle index 21f7734..47e0f09 100644 --- a/elx-api/build.gradle +++ b/elx-api/build.gradle @@ -17,4 +17,6 @@ dependencies { compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson.version')}" // not used, but maybe in other projects compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson.version')}" + // lift guava + compile "com.google.guava:guava:${project.property('guava.version')}" } \ No newline at end of file diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index de16805..dea431c 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -547,10 +547,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String resolveMostRecentIndex(String alias) { - ensureActive(); if (alias == null) { return null; } + ensureActive(); GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias); GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); diff --git a/gradle.properties b/gradle.properties index f941f54..519fda6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,20 +1,22 @@ group = org.xbib name = elx -version = 2.2.1.10 +version = 2.2.1.11 +# main xbib-metrics.version = 1.2.0 xbib-guice.version = 4.0.4 -xbib-netty-http.version = 4.1.36.7 - +guava.version = 28.0-jre +xbib-netty-http.version = 4.1.38.3 elasticsearch.version = 2.2.1 -jackson.version = 2.6.7 +#jackson.version = 2.6.7 +jackson.version = 2.9.9 jna.version = 4.5.2 -log4j.version = 2.11.1 +log4j.version = 2.12.0 mustache.version = 0.9.5 jts.version = 1.13 # test -junit.version = 5.4.2 +junit.version = 5.5.1 # docs asciidoclet.version = 1.5.4 From 8d6cb72777d9a656c7cd8defb082cf821bbb85a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 7 Nov 2019 16:47:47 +0100 Subject: [PATCH 14/19] update to nett 4.1.43, gradle 5.6.2, jackson 2.9.10, metrics 2.0.0 --- build.gradle | 22 ++++++++-------------- gradle.properties | 16 +++++++--------- gradle/wrapper/gradle-wrapper.properties | 6 +++--- gradlew | 6 +++--- gradlew.bat | 2 +- 5 files changed, 22 insertions(+), 30 deletions(-) diff --git a/build.gradle b/build.gradle index 635759c..2f8e691 100644 --- a/build.gradle +++ b/build.gradle @@ -1,21 +1,16 @@ plugins { - id "org.sonarqube" version "2.6.1" - id "io.codearte.nexus-staging" version "0.11.0" - id "com.github.spotbugs" version "1.6.9" + id "org.sonarqube" version "2.8" + id "io.codearte.nexus-staging" version "0.21.1" + id "com.github.spotbugs" version "2.0.1" id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" } -if (JavaVersion.current() < JavaVersion.VERSION_11) { - throw new GradleException("The build must be run with Java 11") -} - subprojects { apply plugin: 'java' apply plugin: 'maven' - apply plugin: 'signing' - apply plugin: 'com.github.spotbugs' apply plugin: 'pmd' apply plugin: 'checkstyle' + apply plugin: 'com.github.spotbugs' apply plugin: 'org.xbib.gradle.plugin.asciidoctor' configurations { @@ -203,9 +198,8 @@ subprojects { } } } +} - nexusStaging { - packageGroup = "org.xbib" - } - -} \ No newline at end of file +nexusStaging { + packageGroup = "org.xbib" +} diff --git a/gradle.properties b/gradle.properties index 519fda6..9bb1cbb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,17 +1,17 @@ group = org.xbib name = elx -version = 2.2.1.11 +version = 2.2.1.13 # main -xbib-metrics.version = 1.2.0 +xbib-metrics.version = 2.0.0 xbib-guice.version = 4.0.4 -guava.version = 28.0-jre -xbib-netty-http.version = 4.1.38.3 +xbib-netty-http.version = 4.1.43.1 elasticsearch.version = 2.2.1 -#jackson.version = 2.6.7 -jackson.version = 2.9.9 +guava.version = 28.0-jre +#jackson.version = 2.6.7 (original ES version) +jackson.version = 2.9.10 jna.version = 4.5.2 -log4j.version = 2.12.0 +log4j.version = 2.12.1 mustache.version = 0.9.5 jts.version = 1.13 @@ -20,5 +20,3 @@ junit.version = 5.5.1 # docs asciidoclet.version = 1.5.4 - -org.gradle.warning.mode = all diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index f3a4f28..f2ac731 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Tue Jul 23 11:33:24 CEST 2019 +#Thu Nov 07 16:18:04 CET 2019 +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.3.1-all.zip +zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index b0d6d0a..83f2acf 100755 --- a/gradlew +++ b/gradlew @@ -7,7 +7,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -125,8 +125,8 @@ if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` diff --git a/gradlew.bat b/gradlew.bat index 15e1ee3..24467a1 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -5,7 +5,7 @@ @rem you may not use this file except in compliance with the License. @rem You may obtain a copy of the License at @rem -@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem https://www.apache.org/licenses/LICENSE-2.0 @rem @rem Unless required by applicable law or agreed to in writing, software @rem distributed under the License is distributed on an "AS IS" BASIS, From 2c6617473dba40752cdd17fda19c185c0794da34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Sun, 19 Jan 2020 22:20:00 +0100 Subject: [PATCH 15/19] change to our guava/guice --- elx-api/build.gradle | 4 +++- gradle.properties | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/elx-api/build.gradle b/elx-api/build.gradle index 47e0f09..41bb17d 100644 --- a/elx-api/build.gradle +++ b/elx-api/build.gradle @@ -7,6 +7,8 @@ dependencies { exclude module: 'securesm' // we use log4j2, not log4j exclude group: 'log4j' + // we use our own guava + exclude group: 'com.google.guava' } // override log4j2 of Elastic with ours compile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" @@ -18,5 +20,5 @@ dependencies { // not used, but maybe in other projects compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson.version')}" // lift guava - compile "com.google.guava:guava:${project.property('guava.version')}" + compile "org.xbib:guava:${project.property('xbib-guava.version')}" } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 9bb1cbb..67e94f2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,14 +1,16 @@ group = org.xbib name = elx -version = 2.2.1.13 +version = 2.2.1.14 # main xbib-metrics.version = 2.0.0 -xbib-guice.version = 4.0.4 -xbib-netty-http.version = 4.1.43.1 +# guice 4 - > our guice 4.4.2 +xbib-guice.version = 4.4.2 +# guava 18 -> our guava 28.1 +xbib-guava.version = 28.1 +xbib-netty-http.version = 4.1.45.0 elasticsearch.version = 2.2.1 -guava.version = 28.0-jre -#jackson.version = 2.6.7 (original ES version) +#jackson 2.6.7 (original ES version) -> 2.9.10 jackson.version = 2.9.10 jna.version = 4.5.2 log4j.version = 2.12.1 From 670245298269232c9e433f3009c017eee52fbe25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Tue, 31 Mar 2020 22:18:15 +0200 Subject: [PATCH 16/19] update to Netty 4.1.48 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 67e94f2..4b70d38 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.14 +version = 2.2.1.15 # main xbib-metrics.version = 2.0.0 @@ -8,7 +8,7 @@ xbib-metrics.version = 2.0.0 xbib-guice.version = 4.4.2 # guava 18 -> our guava 28.1 xbib-guava.version = 28.1 -xbib-netty-http.version = 4.1.45.0 +xbib-netty-http.version = 4.1.48.0 elasticsearch.version = 2.2.1 #jackson 2.6.7 (original ES version) -> 2.9.10 jackson.version = 2.9.10 From 539c77cbd4e1e649f2013c8b3d5fadd21bbf7ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Mon, 11 May 2020 23:49:55 +0200 Subject: [PATCH 17/19] update to elx API of 7.6 branch --- .travis.yml | 9 - build.gradle | 21 +- .../java/org/xbib/elx/api/BulkController.java | 13 +- .../java/org/xbib/elx/api/BulkListener.java | 43 +++ .../java/org/xbib/elx/api/BulkMetric.java | 3 - .../java/org/xbib/elx/api/BulkProcessor.java | 49 +--- .../org/xbib/elx/api/BulkRequestHandler.java | 11 + .../java/org/xbib/elx/api/ExtendedClient.java | 15 +- .../org/xbib/elx/api/IndexPruneResult.java | 2 +- .../org/xbib/elx/api/ReadClientProvider.java | 1 - .../elx/common/AbstractExtendedClient.java | 256 +++++++++++------- .../org/xbib/elx/common/ClientBuilder.java | 11 +- .../elx/common/DefaultBulkController.java | 164 +++-------- .../xbib/elx/common/DefaultBulkListener.java | 109 ++++++++ .../xbib/elx/common/DefaultBulkMetric.java | 5 - .../xbib/elx/common/DefaultBulkProcessor.java | 70 ++--- .../xbib/elx/common/MockExtendedClient.java | 2 +- .../org/xbib/elx/common/test/AliasTest.java | 3 +- .../org/xbib/elx/common/test/SearchTest.java | 2 +- .../org/xbib/elx/common/test/SimpleTest.java | 3 +- .../xbib/elx/common/test/TestExtension.java | 121 ++++----- .../xbib/elx/common/test/WildcardTest.java | 17 +- .../xbib/elx/common/test/package-info.java | 2 +- .../org/xbib/elx/node/test/ClientTest.java | 6 +- .../xbib/elx/node/test/DuplicateIDTest.java | 2 +- .../org/xbib/elx/node/test/SmokeTest.java | 6 +- .../org/xbib/elx/node/test/TestExtension.java | 133 ++++----- .../xbib/elx/transport/test/ClientTest.java | 6 +- .../elx/transport/test/DuplicateIDTest.java | 2 +- .../xbib/elx/transport/test/SmokeTest.java | 6 +- .../elx/transport/test/TestExtension.java | 139 +++++----- gradle.properties | 9 +- 32 files changed, 650 insertions(+), 591 deletions(-) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkListener.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkRequestHandler.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java diff --git a/.travis.yml b/.travis.yml index 0c75adf..94d2a22 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,3 @@ language: java -sudo: required jdk: - openjdk11 -cache: - directories: - - $HOME/.m2 -after_success: - - ./gradlew sonarqube -Dsonar.host.url=https://sonarqube.com -Dsonar.login=$SONAR_TOKEN -env: - global: - secure: n1Ai4q/yMLn/Pg5pA4lTavoJoe7mQYB1PSKnZAqwbgyla94ySzK6iyBCBiNs/foMPisB/x+DHvmUXTsjvquw9Ay48ZITCV3xhcWzD0eZM2TMoG19CpRAEe8L8LNuYiti9k89ijDdUGZ5ifsvQNTGNHksouayAuApC3PrTUejJfR6SYrp1ZsQTbsMlr+4XU3p7QknK5rGgOwATIMP28F+bVnB05WJtlJA3b0SeucCurn3wJ4FGBQXRYmdlT7bQhNE4QgZM1VzcUFD/K0TBxzzq/otb/lNRSifyoekktDmJwQnaT9uQ4R8R6KdQ2Kb38Rvgjur+TKm5i1G8qS2+6LnIxQJG1aw3JvKK6W0wWCgnAVVRrXaCLday9NuY59tuh1mfjQ10UcsMNKcTdcKEMrLow506wSETcXc7L/LEnneWQyJJeV4vhPqR7KJfsBbeqgz3yIfsCn1GZVWFlfegzYCN52YTl0Y0uRD2Z+TnzQu+Bf4DzaWXLge1rz31xkhyeNNspub4h024+XqBjcMm6M9mlMzmmK8t2DIwPy/BlQbFBUyhrxziuR/5/2NEDPyHltvWkRb4AUIa25WJqkV0gTBegbMadZ9DyOo6Ea7aoVFBae2WGR08F1kzABsWrd1S7UJmWxW35iyMEtoAIayXphIK98qO5aCutwZ+3iOQazxbAs= diff --git a/build.gradle b/build.gradle index 2f8e691..60e581e 100644 --- a/build.gradle +++ b/build.gradle @@ -2,7 +2,11 @@ plugins { id "org.sonarqube" version "2.8" id "io.codearte.nexus-staging" version "0.21.1" id "com.github.spotbugs" version "2.0.1" - id "org.xbib.gradle.plugin.asciidoctor" version "1.5.6.0.1" + id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1" +} + +if (JavaVersion.current() < JavaVersion.VERSION_11) { + throw new GradleException("This build must be run with Java/OpenJDK 11+") } subprojects { @@ -19,10 +23,10 @@ subprojects { dependencies { testImplementation "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" testImplementation "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}" testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" } @@ -46,14 +50,11 @@ subprojects { test { enabled = true useJUnitPlatform() - // we MUST use this hack because of Elasticsearch 2.2.1 Lucene 5.4.1 MMapDirectory unmap() hackery - doFirst { - jvmArgs = [ - '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', - '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED' - ] - } + jvmArgs = [ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' systemProperty 'jna.debug_load', 'true' systemProperty 'path.home', "${project.buildDir}" diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkController.java b/elx-api/src/main/java/org/xbib/elx/api/BulkController.java index 69906ca..ec375eb 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkController.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkController.java @@ -14,6 +14,10 @@ public interface BulkController extends Closeable, Flushable { void init(Settings settings); + void inactivate(); + + BulkMetric getBulkMetric(); + Throwable getLastBulkError(); void startBulkMode(IndexDefinition indexDefinition) throws IOException; @@ -21,16 +25,15 @@ public interface BulkController extends Closeable, Flushable { void startBulkMode(String indexName, long startRefreshIntervalInSeconds, long stopRefreshIntervalInSeconds) throws IOException; - void index(IndexRequest indexRequest); + void bulkIndex(IndexRequest indexRequest); - void delete(DeleteRequest deleteRequest); + void bulkDelete(DeleteRequest deleteRequest); - void update(UpdateRequest updateRequest); + void bulkUpdate(UpdateRequest updateRequest); - boolean waitForResponses(long timeout, TimeUnit timeUnit); + boolean waitForBulkResponses(long timeout, TimeUnit timeUnit); void stopBulkMode(IndexDefinition indexDefinition) throws IOException; void stopBulkMode(String index, long timeout, TimeUnit timeUnit) throws IOException; - } diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java b/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java new file mode 100644 index 0000000..9caa339 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkListener.java @@ -0,0 +1,43 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; + +public interface BulkListener { + + /** + * Callback before the bulk is executed. + * + * @param executionId execution ID + * @param request request + */ + void beforeBulk(long executionId, BulkRequest request); + + /** + * Callback after a successful execution of bulk request. + * + * @param executionId execution ID + * @param request request + * @param response response + */ + void afterBulk(long executionId, BulkRequest request, BulkResponse response); + + /** + * Callback after a failed execution of bulk request. + * + * Note that in case an instance of InterruptedException is passed, which means that request + * processing has been + * cancelled externally, the thread's interruption status has been restored prior to calling this method. + * + * @param executionId execution ID + * @param request request + * @param failure failure + */ + void afterBulk(long executionId, BulkRequest request, Throwable failure); + + /** + * Get the last bulk error. + * @return the last bulk error + */ + Throwable getLastBulkError(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java index af825e5..7e84376 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java @@ -1,6 +1,5 @@ package org.xbib.elx.api; -import org.elasticsearch.common.settings.Settings; import org.xbib.metrics.api.Count; import org.xbib.metrics.api.Metered; @@ -8,8 +7,6 @@ import java.io.Closeable; public interface BulkMetric extends Closeable { - void init(Settings settings); - Metered getTotalIngest(); Count getTotalIngestSizeInBytes(); diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java index cb994e0..73cc462 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkProcessor.java @@ -1,8 +1,6 @@ package org.xbib.elx.api; import org.elasticsearch.action.ActionRequest; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import java.io.Closeable; import java.io.Flushable; @@ -13,54 +11,9 @@ public interface BulkProcessor extends Closeable, Flushable { @SuppressWarnings("rawtypes") BulkProcessor add(ActionRequest request); - @SuppressWarnings("rawtypes") - BulkProcessor add(ActionRequest request, Object payload); - boolean awaitFlush(long timeout, TimeUnit unit) throws InterruptedException; boolean awaitClose(long timeout, TimeUnit unit) throws InterruptedException; - interface BulkRequestHandler { - - void execute(BulkRequest bulkRequest, long executionId); - - boolean close(long timeout, TimeUnit unit) throws InterruptedException; - - } - - /** - * A listener for the execution. - */ - interface Listener { - - /** - * Callback before the bulk is executed. - * - * @param executionId execution ID - * @param request request - */ - void beforeBulk(long executionId, BulkRequest request); - - /** - * Callback after a successful execution of bulk request. - * - * @param executionId execution ID - * @param request request - * @param response response - */ - void afterBulk(long executionId, BulkRequest request, BulkResponse response); - - /** - * Callback after a failed execution of bulk request. - * - * Note that in case an instance of InterruptedException is passed, which means that request - * processing has been - * cancelled externally, the thread's interruption status has been restored prior to calling this method. - * - * @param executionId execution ID - * @param request request - * @param failure failure - */ - void afterBulk(long executionId, BulkRequest request, Throwable failure); - } + BulkListener getBulkListener(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkRequestHandler.java b/elx-api/src/main/java/org/xbib/elx/api/BulkRequestHandler.java new file mode 100644 index 0000000..1bc3886 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkRequestHandler.java @@ -0,0 +1,11 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.bulk.BulkRequest; +import java.util.concurrent.TimeUnit; + +public interface BulkRequestHandler { + + void execute(BulkRequest bulkRequest, long executionId); + + boolean close(long timeout, TimeUnit unit) throws InterruptedException; +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index 0ec8639..6114621 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -6,6 +6,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.Closeable; import java.io.Flushable; @@ -34,12 +35,6 @@ public interface ExtendedClient extends Flushable, Closeable { */ ElasticsearchClient getClient(); - /** - * Get bulk metric. - * @return the bulk metric - */ - BulkMetric getBulkMetric(); - /** * Get buulk control. * @return the bulk control @@ -190,6 +185,8 @@ public interface ExtendedClient extends Flushable, Closeable { */ ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException; + ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; + /** * Create a new index. * @@ -199,7 +196,7 @@ public interface ExtendedClient extends Flushable, Closeable { * @return this * @throws IOException if settings/mapping is invalid or index creation fails */ - ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; + ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; /** * Create a new index. @@ -317,7 +314,7 @@ public interface ExtendedClient extends Flushable, Closeable { /** * Force segment merge of an index. - * @param indexDefinition th eindex definition + * @param indexDefinition the index definition * @return this */ boolean forceMerge(IndexDefinition indexDefinition); @@ -398,7 +395,7 @@ public interface ExtendedClient extends Flushable, Closeable { String resolveMostRecentIndex(String alias); /** - * Get all aliases. + * Get all index aliases. * @param index the index * @return map of index aliases */ diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java index 0c118f8..a4ef207 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java @@ -4,7 +4,7 @@ import java.util.List; public interface IndexPruneResult { - enum State { NOTHING_TO_DO, SUCCESS, NONE }; + enum State { NOTHING_TO_DO, SUCCESS, NONE, FAIL }; State getState(); diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java index 6640686..bc0eb16 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java @@ -1,6 +1,5 @@ package org.xbib.elx.api; -@FunctionalInterface public interface ReadClientProvider { C getReadClient(); diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java index dea431c..8ccb489 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java @@ -2,6 +2,7 @@ package org.xbib.elx.common; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchTimeoutException; @@ -58,6 +59,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -67,7 +69,6 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.xbib.elx.api.BulkController; -import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexAliasAdder; import org.xbib.elx.api.IndexDefinition; @@ -85,7 +86,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -108,22 +108,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); - /** - * The one and only index type name used in the extended client. - * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". - */ - private static final String TYPE_NAME = "doc"; - - /** - * The Elasticsearch client. - */ private ElasticsearchClient client; - private BulkMetric bulkMetric; - private BulkController bulkController; - private AtomicBoolean closed; + private final AtomicBoolean closed; private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { @Override @@ -178,11 +167,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return client; } - @Override - public BulkMetric getBulkMetric() { - return bulkMetric; - } - @Override public BulkController getBulkController() { return bulkController; @@ -190,15 +174,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public AbstractExtendedClient init(Settings settings) throws IOException { + logger.info("initializing with settings = " + settings.toDelimitedString(',')); if (client == null) { client = createClient(settings); } - if (bulkMetric == null) { - this.bulkMetric = new DefaultBulkMetric(); - this.bulkMetric.init(settings); - } if (bulkController == null) { - this.bulkController = new DefaultBulkController(this, bulkMetric); + this.bulkController = new DefaultBulkController(this); bulkController.init(settings); } return this; @@ -213,12 +194,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public void close() throws IOException { - ensureActive(); + ensureClient(); if (closed.compareAndSet(false, true)) { - if (bulkMetric != null) { - logger.info("closing bulk metric"); - bulkMetric.close(); - } if (bulkController != null) { logger.info("closing bulk controller"); bulkController.close(); @@ -229,9 +206,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String getClusterName() { - ensureActive(); + ensureClient(); try { - ClusterStateRequest clusterStateRequest = new ClusterStateRequest().all(); + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); ClusterStateResponse clusterStateResponse = client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); return clusterStateResponse.getClusterName().value(); @@ -249,7 +226,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException { - ensureActive(); + ensureClient(); waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); URL indexSettings = indexDefinition.getSettingsUrl(); if (indexSettings == null) { @@ -284,7 +261,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(String index) throws IOException { - return newIndex(index, Settings.EMPTY, (Map) null); + return newIndex(index, Settings.EMPTY, (Map) null); } @Override @@ -296,7 +273,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(String index, Settings settings) throws IOException { - return newIndex(index, settings, (Map) null); + return newIndex(index, settings, (Map) null); } @Override @@ -306,8 +283,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) { - ensureActive(); + public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) { + ensureClient(); if (index == null) { logger.warn("no index name given to create index"); return this; @@ -317,11 +294,35 @@ public abstract class AbstractExtendedClient implements ExtendedClient { createIndexRequest.settings(settings); } if (mapping != null) { - createIndexRequest.mapping(TYPE_NAME, mapping); + createIndexRequest.mapping("doc", mapping); } CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - logger.info("index {} created: {}", index, createIndexResponse); - return this; + if (createIndexResponse.isAcknowledged()) { + return this; + } + throw new IllegalStateException("index creation not acknowledged: " + index); + } + + + @Override + public ExtendedClient newIndex(String index, Settings settings, Map mapping) { + ensureClient(); + if (index == null) { + logger.warn("no index name given to create index"); + return this; + } + CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); + if (settings != null) { + createIndexRequest.settings(settings); + } + if (mapping != null) { + createIndexRequest.mapping("doc", mapping); + } + CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + if (createIndexResponse.isAcknowledged()) { + return this; + } + throw new IllegalStateException("index creation not acknowledged: " + index); } @Override @@ -331,7 +332,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient deleteIndex(String index) { - ensureActive(); + ensureClient(); if (index == null) { logger.warn("no index name given to delete index"); return this; @@ -351,7 +352,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException { if (bulkController != null) { - ensureActive(); + ensureClient(); bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); } return this; @@ -360,7 +361,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException { if (bulkController != null) { - ensureActive(); + ensureClient(); bulkController.stopBulkMode(indexDefinition); } return this; @@ -369,7 +370,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { if (bulkController != null) { - ensureActive(); + ensureClient(); bulkController.stopBulkMode(index, timeout, timeUnit); } return this; @@ -377,63 +378,63 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient index(String index, String id, boolean create, String source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) + return index(new IndexRequest().index(index).type("doc").id(id).create(create) .source(source.getBytes(StandardCharsets.UTF_8))); } @Override public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) + return index(new IndexRequest().index(index).type("doc").id(id).create(create) .source(source)); } @Override public ExtendedClient index(IndexRequest indexRequest) { - ensureActive(); - bulkController.index(indexRequest); + ensureClient(); + bulkController.bulkIndex(indexRequest); return this; } @Override public ExtendedClient delete(String index, String id) { - return delete(new DeleteRequest(index, TYPE_NAME, id)); + return delete(new DeleteRequest().index(index).type("doc").id(id)); } @Override public ExtendedClient delete(DeleteRequest deleteRequest) { - ensureActive(); - bulkController.delete(deleteRequest); + ensureClient(); + bulkController.bulkDelete(deleteRequest); return this; } @Override public ExtendedClient update(String index, String id, BytesReference source) { - return update(new UpdateRequest(index, TYPE_NAME, id) + return update(new UpdateRequest().index(index).type("doc").id(id) .doc(source)); } @Override public ExtendedClient update(String index, String id, String source) { - return update(new UpdateRequest(index, TYPE_NAME, id) + return update(new UpdateRequest().index(index).type("doc").id(id) .doc(source.getBytes(StandardCharsets.UTF_8))); } @Override public ExtendedClient update(UpdateRequest updateRequest) { - ensureActive(); - bulkController.update(updateRequest); + ensureClient(); + bulkController.bulkUpdate(updateRequest); return this; } @Override public boolean waitForResponses(long timeout, TimeUnit timeUnit) { - ensureActive(); - return bulkController.waitForResponses(timeout, timeUnit); + ensureClient(); + return bulkController.waitForBulkResponses(timeout, timeUnit); } @Override public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - ensureActive(); + ensureClient(); ensureIndexGiven(index); GetSettingsRequest settingsRequest = new GetSettingsRequest(); settingsRequest.indices(index); @@ -448,7 +449,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - logger.error("timeout waiting for recovery"); + logger.warn("timeout waiting for recovery"); return false; } } @@ -457,15 +458,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { - ensureActive(); + ensureClient(); ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - if (logger.isErrorEnabled()) { - logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); - } + logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); return false; } return true; @@ -473,7 +472,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { - ensureActive(); + ensureClient(); try { TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, @@ -530,7 +529,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient flushIndex(String index) { if (index != null) { - ensureActive(); + ensureClient(); client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); } return this; @@ -539,7 +538,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient refreshIndex(String index) { if (index != null) { - ensureActive(); + ensureClient(); client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); } return this; @@ -550,7 +549,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (alias == null) { return null; } - ensureActive(); + ensureClient(); GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias); GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); @@ -574,9 +573,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String resolveAlias(String alias) { - ensureActive(); + ensureClient(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); + clusterStateRequest.blocks(false); clusterStateRequest.metaData(true); + clusterStateRequest.nodes(false); + clusterStateRequest.routingTable(false); + clusterStateRequest.customs(false); ClusterStateResponse clusterStateResponse = client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); SortedMap map = clusterStateResponse.getState().getMetaData().getAliasAndIndexLookup(); @@ -612,7 +615,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, IndexAliasAdder adder) { - ensureActive(); + ensureClient(); if (index == null) { return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to } @@ -706,11 +709,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (index.equals(fullIndexName)) { return EMPTY_INDEX_PRUNE_RESULT; } - ensureActive(); + ensureClient(); GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - logger.info("{} indices", getIndexResponse.getIndices().length); + logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length); List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); @@ -746,20 +749,29 @@ public abstract class AbstractExtendedClient implements ExtendedClient { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); DeleteIndexResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); - return new SuccessPruneResult(candidateIndices, indicesToDelete, response); + if (response.isAcknowledged()) { + logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s)); + waitForCluster("GREEN", 30L, TimeUnit.SECONDS); + return new SuccessPruneResult(candidateIndices, indicesToDelete, response); + } else { + logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s)); + return new FailPruneResult(candidateIndices, indicesToDelete, response); + } } @Override public Long mostRecentDocument(String index, String timestampfieldname) { - ensureActive(); - SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); - sourceBuilder.field(timestampfieldname); - sourceBuilder.size(1); - sourceBuilder.sort(sort); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices(index); - searchRequest.source(sourceBuilder); + ensureClient(); + SortBuilder sort = SortBuilders + .fieldSort(timestampfieldname) + .order(SortOrder.DESC); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .sort(sort) + .field(timestampfieldname) + .size(1); + SearchRequest searchRequest = new SearchRequest() + .indices(index) + .source(sourceBuilder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); if (searchResponse.getHits().getHits().length == 1) { SearchHit hit = searchResponse.getHits().getHits()[0]; @@ -837,7 +849,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { - ensureActive(); + ensureClient(); if (index == null) { throw new IOException("no index name given"); } @@ -854,7 +866,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); } - private void ensureActive() { + private void ensureClient() { if (this instanceof MockExtendedClient) { return; } @@ -886,7 +898,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } public void checkMapping(String index) { - ensureActive(); + ensureClient(); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index); GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); ImmutableOpenMap> map = getMappingsResponse.getMappings(); @@ -902,25 +914,24 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { try { - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); - SearchResponse searchResponse = searchRequestBuilder.setSize(0) - .setIndices(index) - .setTypes(type) - .setQuery(QueryBuilders.matchAllQuery()) - .execute() - .actionGet(); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(QueryBuilders.matchAllQuery()) + .size(0); + SearchRequest searchRequest = new SearchRequest() + .indices(index) + .source(builder); + SearchResponse searchResponse = + client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); long total = searchResponse.getHits().getTotalHits(); if (total > 0L) { Map fields = new TreeMap<>(); Map root = mappingMetaData.getSourceAsMap(); - checkMapping(index, type, "", "", root, fields); + checkMapping(index, "", "", root, fields); AtomicInteger empty = new AtomicInteger(); Map map = sortByValue(fields); map.forEach((key, value) -> { logger.info("{} {} {}", - key, - value, - (double) value * 100 / total); + key, value, (double) value * 100 / total); if (value == 0) { empty.incrementAndGet(); } @@ -934,7 +945,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @SuppressWarnings("unchecked") - private void checkMapping(String index, String type, + private void checkMapping(String index, String pathDef, String fieldName, Map map, Map fields) { String path = pathDef; @@ -959,18 +970,19 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String fieldType = o instanceof String ? o.toString() : null; // do not recurse into our custom field mapper if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) { - checkMapping(index, type, path, key, child, fields); + checkMapping(index, path, key, child, fields); } } else if ("type".equals(key)) { QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); - SearchResponse searchResponse = searchRequestBuilder.setSize(0) - .setIndices(index) - .setTypes(type) - .setQuery(queryBuilder) - .execute() - .actionGet(); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(queryBuilder) + .size(0); + SearchRequest searchRequest = new SearchRequest() + .indices(index) + .source(builder); + SearchResponse searchResponse = + client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); fields.put(path, searchResponse.getHits().getTotalHits()); } } @@ -978,7 +990,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static > Map sortByValue(Map map) { Map result = new LinkedHashMap<>(); - map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue)) + map.entrySet().stream().sorted(Map.Entry.comparingByValue()) .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); return result; } @@ -1062,6 +1074,42 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } } + private static class FailPruneResult implements IndexPruneResult { + + List candidateIndices; + + List indicesToDelete; + + DeleteIndexResponse response; + + FailPruneResult(List candidateIndices, List indicesToDelete, + DeleteIndexResponse response) { + this.candidateIndices = candidateIndices; + this.indicesToDelete = indicesToDelete; + this.response = response; + } + + @Override + public IndexPruneResult.State getState() { + return IndexPruneResult.State.FAIL; + } + + @Override + public List getCandidateIndices() { + return candidateIndices; + } + + @Override + public List getDeletedIndices() { + return indicesToDelete; + } + + @Override + public boolean isAcknowledged() { + return response.isAcknowledged(); + } + } + private static class NothingToDoPruneResult implements IndexPruneResult { List candidateIndices; diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java index ba9150f..e77a55a 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -1,5 +1,8 @@ package org.xbib.elx.common; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; @@ -16,6 +19,8 @@ import java.util.ServiceLoader; @SuppressWarnings("rawtypes") public class ClientBuilder { + private static final Logger logger = LogManager.getLogger(ClientBuilder.class); + private final ElasticsearchClient client; private final Settings.Builder settingsBuilder; @@ -97,6 +102,10 @@ public class ClientBuilder { if (provider == null) { throw new IllegalArgumentException("no provider"); } - return (C) providerMap.get(provider).getExtendedClient().setClient(client).init(settingsBuilder.build()); + Settings settings = settingsBuilder.build(); + logger.log(Level.INFO, "settings = " + settings.toDelimitedString(',')); + return (C) providerMap.get(provider).getExtendedClient() + .setClient(client) + .init(settings); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index dbd7fa6..41c3e8b 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -2,9 +2,6 @@ package org.xbib.elx.common; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.bulk.BulkItemResponse; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; @@ -12,6 +9,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkListener; import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkProcessor; import org.xbib.elx.api.ExtendedClient; @@ -33,27 +31,23 @@ public class DefaultBulkController implements BulkController { private final BulkMetric bulkMetric; + private BulkProcessor bulkProcessor; + private final List indexNames; private final Map startBulkRefreshIntervals; private final Map stopBulkRefreshIntervals; - private long maxWaitTime; + private final long maxWaitTime; - private TimeUnit maxWaitTimeUnit; + private final TimeUnit maxWaitTimeUnit; - private BulkProcessor bulkProcessor; + private final AtomicBoolean active; - private BulkListener bulkListener; - - private AtomicBoolean active; - - private boolean enableBulkLogging; - - public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) { + public DefaultBulkController(ExtendedClient client) { this.client = client; - this.bulkMetric = bulkMetric; + this.bulkMetric = new DefaultBulkMetric(); this.indexNames = new ArrayList<>(); this.active = new AtomicBoolean(false); this.startBulkRefreshIntervals = new HashMap<>(); @@ -62,9 +56,14 @@ public class DefaultBulkController implements BulkController { this.maxWaitTimeUnit = TimeUnit.SECONDS; } + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + @Override public Throwable getLastBulkError() { - return bulkListener.getLastBulkError(); + return bulkProcessor.getBulkListener().getLastBulkError(); } @Override @@ -78,22 +77,27 @@ public class DefaultBulkController implements BulkController { ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), "maxVolumePerRequest")); - this.enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), + boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), Parameters.ENABLE_BULK_LOGGING.getValue()); - this.bulkListener = new BulkListener(); + BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging); this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) .setConcurrentRequests(maxConcurrentRequests) .setFlushInterval(flushIngestInterval) .setBulkSize(maxVolumePerRequest) .build(); - this.active.set(true); if (logger.isInfoEnabled()) { - logger.info("bulk processor set up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + + logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + "flushIngestInterval = {} maxVolumePerRequest = {}, bulk logging = {}", maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest, enableBulkLogging); } + this.active.set(true); + } + + @Override + public void inactivate() { + this.active.set(false); } @Override @@ -118,65 +122,53 @@ public class DefaultBulkController implements BulkController { } @Override - public void index(IndexRequest indexRequest) { + public void bulkIndex(IndexRequest indexRequest) { ensureActiveAndBulk(); - if (!active.get()) { - throw new IllegalStateException("inactive"); - } try { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); - } + bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); bulkProcessor.add(indexRequest); } catch (Exception e) { - bulkListener.lastBulkError = e; - active.set(false); if (logger.isErrorEnabled()) { logger.error("bulk add of index failed: " + e.getMessage(), e); } + inactivate(); } } @Override - public void delete(DeleteRequest deleteRequest) { + public void bulkDelete(DeleteRequest deleteRequest) { if (!active.get()) { throw new IllegalStateException("inactive"); } try { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); - } + bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); bulkProcessor.add(deleteRequest); } catch (Exception e) { - bulkListener.lastBulkError = e; - active.set(false); if (logger.isErrorEnabled()) { logger.error("bulk add of delete failed: " + e.getMessage(), e); } + inactivate(); } } @Override - public void update(UpdateRequest updateRequest) { + public void bulkUpdate(UpdateRequest updateRequest) { if (!active.get()) { throw new IllegalStateException("inactive"); } try { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); - } + bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); bulkProcessor.add(updateRequest); } catch (Exception e) { - bulkListener.lastBulkError = e; - active.set(false); if (logger.isErrorEnabled()) { logger.error("bulk add of update failed: " + e.getMessage(), e); } + inactivate(); } } @Override - public boolean waitForResponses(long timeout, TimeUnit timeUnit) { + public boolean waitForBulkResponses(long timeout, TimeUnit timeUnit) { try { return bulkProcessor.awaitFlush(timeout, timeUnit); } catch (InterruptedException e) { @@ -195,7 +187,7 @@ public class DefaultBulkController implements BulkController { @Override public void stopBulkMode(String index, long timeout, TimeUnit timeUnit) throws IOException { flush(); - if (waitForResponses(timeout, timeUnit)) { + if (waitForBulkResponses(timeout, timeUnit)) { if (indexNames.contains(index)) { Long secs = stopBulkRefreshIntervals.get(index); if (secs != null && secs != 0L) { @@ -217,6 +209,7 @@ public class DefaultBulkController implements BulkController { @Override public void close() throws IOException { flush(); + bulkMetric.close(); if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) { for (String index : indexNames) { Long secs = stopBulkRefreshIntervals.get(index); @@ -238,92 +231,5 @@ public class DefaultBulkController implements BulkController { if (bulkProcessor == null) { throw new UnsupportedOperationException("bulk processor not present"); } - if (bulkListener == null) { - throw new UnsupportedOperationException("bulk listener not present"); - } - } - - private class BulkListener implements DefaultBulkProcessor.Listener { - - private final Logger logger = LogManager.getLogger(BulkListener.class.getName()); - - private Throwable lastBulkError = null; - - @Override - public void beforeBulk(long executionId, BulkRequest request) { - long l = 0; - if (bulkMetric != null) { - l = bulkMetric.getCurrentIngest().getCount(); - bulkMetric.getCurrentIngest().inc(); - int n = request.numberOfActions(); - bulkMetric.getSubmitted().inc(n); - bulkMetric.getCurrentIngestNumDocs().inc(n); - bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); - } - if (enableBulkLogging && logger.isDebugEnabled()) { - logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", - executionId, - request.numberOfActions(), - request.estimatedSizeInBytes(), - l); - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { - long l = 0; - if (bulkMetric != null) { - l = bulkMetric.getCurrentIngest().getCount(); - bulkMetric.getCurrentIngest().dec(); - bulkMetric.getSucceeded().inc(response.getItems().length); - } - int n = 0; - for (BulkItemResponse itemResponse : response.getItems()) { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); - } - if (itemResponse.isFailed()) { - n++; - if (bulkMetric != null) { - bulkMetric.getSucceeded().dec(1); - bulkMetric.getFailed().inc(1); - } - } - } - if (enableBulkLogging && bulkMetric != null && logger.isDebugEnabled()) { - logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", - executionId, - bulkMetric.getSucceeded().getCount(), - bulkMetric.getFailed().getCount(), - response.getTook().millis(), - l); - } - if (n > 0) { - if (enableBulkLogging && logger.isErrorEnabled()) { - logger.error("bulk [{}] failed with {} failed items, failure message = {}", - executionId, n, response.buildFailureMessage()); - } - } else { - if (bulkMetric != null) { - bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); - } - } - } - - @Override - public void afterBulk(long executionId, BulkRequest request, Throwable failure) { - if (bulkMetric != null) { - bulkMetric.getCurrentIngest().dec(); - } - lastBulkError = failure; - active.set(false); - if (enableBulkLogging && logger.isErrorEnabled()) { - logger.error("after bulk [" + executionId + "] error", failure); - } - } - - Throwable getLastBulkError() { - return lastBulkError; - } } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java new file mode 100644 index 0000000..bb40ba5 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkListener.java @@ -0,0 +1,109 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkListener; +import org.xbib.elx.api.BulkMetric; + +class DefaultBulkListener implements BulkListener { + + private final Logger logger = LogManager.getLogger(BulkListener.class.getName()); + + private final BulkController bulkController; + + private final BulkMetric bulkMetric; + + private final boolean isBulkLoggingEnabled; + + private Throwable lastBulkError = null; + + public DefaultBulkListener(BulkController bulkController, + BulkMetric bulkMetric, + boolean isBulkLoggingEnabled) { + this.bulkController = bulkController; + this.bulkMetric = bulkMetric; + this.isBulkLoggingEnabled = isBulkLoggingEnabled; + } + + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().inc(); + int n = request.numberOfActions(); + bulkMetric.getSubmitted().inc(n); + bulkMetric.getCurrentIngestNumDocs().inc(n); + bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + if (isBulkLoggingEnabled && logger.isDebugEnabled()) { + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().dec(); + bulkMetric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + } + if (itemResponse.isFailed()) { + n++; + if (bulkMetric != null) { + bulkMetric.getSucceeded().dec(1); + bulkMetric.getFailed().inc(1); + } + } + } + if (isBulkLoggingEnabled && bulkMetric != null && logger.isDebugEnabled()) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", + executionId, + bulkMetric.getSucceeded().getCount(), + bulkMetric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + if (isBulkLoggingEnabled && logger.isErrorEnabled()) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } + } else { + if (bulkMetric != null) { + bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(); + } + lastBulkError = failure; + if (logger.isErrorEnabled()) { + logger.error("after bulk [" + executionId + "] error", failure); + } + bulkController.inactivate(); + } + + @Override + public Throwable getLastBulkError() { + return lastBulkError; + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java index 8127e29..1350e65 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java @@ -1,6 +1,5 @@ package org.xbib.elx.common; -import org.elasticsearch.common.settings.Settings; import org.xbib.elx.api.BulkMetric; import org.xbib.metrics.api.Count; import org.xbib.metrics.api.Metered; @@ -37,10 +36,6 @@ public class DefaultBulkMetric implements BulkMetric { submitted = new CountMetric(); succeeded = new CountMetric(); failed = new CountMetric(); - } - - @Override - public void init(Settings settings) { start(); } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java index 74915c9..e815b83 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkProcessor.java @@ -11,7 +11,9 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.xbib.elx.api.BulkListener; import org.xbib.elx.api.BulkProcessor; +import org.xbib.elx.api.BulkRequestHandler; import java.util.Objects; import java.util.concurrent.Executors; @@ -29,6 +31,8 @@ import java.util.concurrent.atomic.AtomicLong; */ public class DefaultBulkProcessor implements BulkProcessor { + private final BulkListener bulkListener; + private final int bulkActions; private final long bulkSize; @@ -45,16 +49,22 @@ public class DefaultBulkProcessor implements BulkProcessor { private volatile boolean closed; - private DefaultBulkProcessor(ElasticsearchClient client, Listener listener, String name, int concurrentRequests, - int bulkActions, ByteSizeValue bulkSize, TimeValue flushInterval) { + private DefaultBulkProcessor(ElasticsearchClient client, + BulkListener bulkListener, + String name, + int concurrentRequests, + int bulkActions, + ByteSizeValue bulkSize, + TimeValue flushInterval) { + this.bulkListener = bulkListener; this.executionIdGen = new AtomicLong(); this.closed = false; this.bulkActions = bulkActions; this.bulkSize = bulkSize.getBytes(); this.bulkRequest = new BulkRequest(); this.bulkRequestHandler = concurrentRequests == 0 ? - new SyncBulkRequestHandler(client, listener) : - new AsyncBulkRequestHandler(client, listener, concurrentRequests); + new SyncBulkRequestHandler(client, bulkListener) : + new AsyncBulkRequestHandler(client, bulkListener, concurrentRequests); if (flushInterval != null) { this.scheduler = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, EsExecutors.daemonThreadFactory(name != null ? "[" + name + "]" : "" + "bulk_processor")); @@ -68,12 +78,18 @@ public class DefaultBulkProcessor implements BulkProcessor { } } - public static Builder builder(ElasticsearchClient client, Listener listener) { + public static Builder builder(ElasticsearchClient client, + BulkListener listener) { Objects.requireNonNull(client, "The client you specified while building a BulkProcessor is null"); Objects.requireNonNull(listener, "A listener for the BulkProcessor is required but null"); return new Builder(client, listener); } + @Override + public BulkListener getBulkListener() { + return bulkListener; + } + /** * Wait for bulk request handler with flush. * @param timeout the timeout value @@ -136,20 +152,7 @@ public class DefaultBulkProcessor implements BulkProcessor { @SuppressWarnings("rawtypes") @Override public DefaultBulkProcessor add(ActionRequest request) { - return add(request, null); - } - - /** - * Adds either a delete or an index request with a payload. - * - * @param request request - * @param payload payload - * @return his bulk processor - */ - @SuppressWarnings("rawtypes") - @Override - public DefaultBulkProcessor add(ActionRequest request, Object payload) { - internalAdd(request, payload); + internalAdd(request); return this; } @@ -183,9 +186,9 @@ public class DefaultBulkProcessor implements BulkProcessor { } } - private synchronized void internalAdd(ActionRequest request, Object payload) { + private synchronized void internalAdd(ActionRequest request) { ensureOpen(); - bulkRequest.add(request, payload); + bulkRequest.add(request); executeIfNeeded(); } @@ -217,7 +220,7 @@ public class DefaultBulkProcessor implements BulkProcessor { private final ElasticsearchClient client; - private final Listener listener; + private final BulkListener bulkListener; private String name; @@ -233,12 +236,12 @@ public class DefaultBulkProcessor implements BulkProcessor { * Creates a builder of bulk processor with the client to use and the listener that will be used * to be notified on the completion of bulk requests. * - * @param client the client - * @param listener the listener + * @param client the client + * @param bulkListener the listener */ - Builder(ElasticsearchClient client, Listener listener) { + Builder(ElasticsearchClient client, BulkListener bulkListener) { this.client = client; - this.listener = listener; + this.bulkListener = bulkListener; } /** @@ -308,7 +311,7 @@ public class DefaultBulkProcessor implements BulkProcessor { * @return a bulk processor */ public DefaultBulkProcessor build() { - return new DefaultBulkProcessor(client, listener, name, concurrentRequests, bulkActions, bulkSize, flushInterval); + return new DefaultBulkProcessor(client, bulkListener, name, concurrentRequests, bulkActions, bulkSize, flushInterval); } } @@ -332,10 +335,9 @@ public class DefaultBulkProcessor implements BulkProcessor { private final ElasticsearchClient client; - private final DefaultBulkProcessor.Listener listener; + private final BulkListener listener; - SyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener) { - Objects.requireNonNull(listener, "A listener is required for SyncBulkRequestHandler but null"); + SyncBulkRequestHandler(ElasticsearchClient client, BulkListener listener) { this.client = client; this.listener = listener; } @@ -365,14 +367,13 @@ public class DefaultBulkProcessor implements BulkProcessor { private final ElasticsearchClient client; - private final DefaultBulkProcessor.Listener listener; + private final BulkListener listener; private final Semaphore semaphore; private final int concurrentRequests; - private AsyncBulkRequestHandler(ElasticsearchClient client, DefaultBulkProcessor.Listener listener, int concurrentRequests) { - Objects.requireNonNull(listener, "A listener is required for AsyncBulkRequestHandler but null"); + private AsyncBulkRequestHandler(ElasticsearchClient client, BulkListener listener, int concurrentRequests) { this.client = client; this.listener = listener; this.concurrentRequests = concurrentRequests; @@ -413,7 +414,8 @@ public class DefaultBulkProcessor implements BulkProcessor { } catch (Exception e) { listener.afterBulk(executionId, bulkRequest, e); } finally { - if (!bulkRequestSetupSuccessful && acquired) { // if we fail on client.bulk() release the semaphore + if (!bulkRequestSetupSuccessful && acquired) { + // if we fail on client.bulk() release the semaphore semaphore.release(); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java index 4f91c14..647894b 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java @@ -29,7 +29,7 @@ public class MockExtendedClient extends AbstractExtendedClient { } @Override - protected void closeClient() { + protected void closeClient() { } @Override diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java index 928dbe6..5c559bb 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/AliasTest.java @@ -52,7 +52,8 @@ class AliasTest { // get alias GetAliasesRequest getAliasesRequest = new GetAliasesRequest(Strings.EMPTY_ARRAY); long t0 = System.nanoTime(); - GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); + GetAliasesResponse getAliasesResponse = + client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); long t1 = (System.nanoTime() - t0) / 1000000; logger.info("{} time(ms) = {}", getAliasesResponse.getAliases(), t1); assertTrue(t1 >= 0); diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java index 2b82483..2bc1f44 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SearchTest.java @@ -33,7 +33,7 @@ class SearchTest { ElasticsearchClient client = helper.client("1"); BulkRequestBuilder builder = new BulkRequestBuilder(client, BulkAction.INSTANCE); for (int i = 0; i < 1; i++) { - IndexRequest indexRequest = new IndexRequest("pages", "row") + IndexRequest indexRequest = new IndexRequest().index("pages").type("row") .source(XContentFactory.jsonBuilder() .startObject() .field("user1", "joerg") diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java index 1ee6bfb..cb56250 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/SimpleTest.java @@ -30,7 +30,7 @@ class SimpleTest { } @Test - void test() throws Exception { + void testSimple() throws Exception { try { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices("test"); @@ -39,7 +39,6 @@ class SimpleTest { // ignore if index not found } Settings indexSettings = Settings.settingsBuilder() - .put(helper.getNodeSettings()) .put("index.analysis.analyzer.default.filter.0", "lowercase") .put("index.analysis.analyzer.default.filter.1", "trim") .put("index.analysis.analyzer.default.tokenizer", "keyword") diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java index 6186f8c..139f560 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/TestExtension.java @@ -36,8 +36,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import java.util.concurrent.atomic.AtomicInteger; public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { @@ -47,20 +46,10 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String home; - - private String cluster; - - private String host; - - private int port; - private static final String key = "es-instance"; + private static final AtomicInteger count = new AtomicInteger(0); + private static final ExtensionContext.Namespace ns = ExtensionContext.Namespace.create(TestExtension.class); @@ -73,20 +62,23 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - setHome(System.getProperty("path.home") + "/" + getRandomString(8)); - setClusterName("test-cluster-" + System.getProperty("user.name")); - return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); } @Override - public void beforeEach(ExtensionContext context) throws Exception { - Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); - logger.info("starting cluster"); + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); helper.startNode("1"); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); Object obj = response.iterator().next().getTransport().getAddress() .publishAddress(); + String host = null; + int port = 0; if (obj instanceof InetSocketTransportAddress) { InetSocketTransportAddress address = (InetSocketTransportAddress) obj; host = address.address().getHostName(); @@ -111,42 +103,26 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft } @Override - public void afterEach(ExtensionContext context) throws Exception { - closeNodes(); - deleteFiles(Paths.get(getHome() + "/data")); + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome() + "/data")); logger.info("data files wiped"); Thread.sleep(2000L); // let OS commit changes } - private void setClusterName(String cluster) { - this.cluster = cluster; - } - - private String getClusterName() { - return cluster; - } - - private void setHome(String home) { - this.home = home; - } - - private String getHome() { - return home; - } - - private void closeNodes() { + private void closeNodes(Helper helper) throws IOException { logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { + for (AbstractClient client : helper.clients.values()) { client.close(); } - clients.clear(); logger.info("closing all nodes"); - for (Node node : nodes.values()) { + for (Node node : helper.nodes.values()) { if (node != null) { node.close(); } } - nodes.clear(); logger.info("all nodes closed"); } @@ -168,23 +144,42 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft } } - private String getRandomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - private Helper create() { - return new Helper(); + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; } - class Helper { + static class Helper { + + String home; + + String cluster; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } Settings getNodeSettings() { - return settingsBuilder() + return Settings.builder() .put("cluster.name", getClusterName()) .put("path.home", getHome()) .build(); @@ -198,20 +193,24 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return clients.get(id); } - String randomString(int n) { - return getRandomString(n); + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); } private Node buildNode(String id) { - Settings nodeSettings = settingsBuilder() + Settings nodeSettings = Settings.builder() .put(getNodeSettings()) - .put("name", id) + .put("node.name", id) .build(); Node node = new MockNode(nodeSettings); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); clients.put(id, client); - logger.info("clients={}", clients); return node; } } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java index 4f298aa..39beedb 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/WildcardTest.java @@ -45,18 +45,19 @@ class WildcardTest { } private void index(ElasticsearchClient client, String id, String fieldValue) throws IOException { - client.execute(IndexAction.INSTANCE, new IndexRequest("index", "type", id) - .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())).actionGet(); + client.execute(IndexAction.INSTANCE, new IndexRequest().index("index").type("type").id(id) + .source(XContentFactory.jsonBuilder().startObject().field("field", fieldValue).endObject())) + .actionGet(); client.execute(RefreshAction.INSTANCE, new RefreshRequest()).actionGet(); } private long count(ElasticsearchClient client, QueryBuilder queryBuilder) { - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(queryBuilder); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("index"); - searchRequest.types("type"); - searchRequest.source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder() + .query(queryBuilder); + SearchRequest searchRequest = new SearchRequest() + .indices("index") + .types("type") + .source(builder); return client.execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); } diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java b/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java index 5a27aff..ab5f171 100644 --- a/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java +++ b/elx-common/src/test/java/org/xbib/elx/common/test/package-info.java @@ -1 +1 @@ -package org.xbib.elx.common.test; \ No newline at end of file +package org.xbib.elx.common.test; diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java index f922bcc..3733275 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java @@ -94,7 +94,7 @@ class ClientTest { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } @@ -122,7 +122,7 @@ class ClientTest { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } @@ -180,7 +180,7 @@ class ClientTest { logger.warn("skipping, no node available"); } finally { client.stopBulk("test5", 60L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index 0ea421c..f2a1689 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -63,7 +63,7 @@ class DuplicateIDTest { logger.warn("skipping, no node available"); } finally { client.close(); - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index c9a7cd7..c0bab0d 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -37,7 +37,7 @@ class SmokeTest { client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - assertEquals(helper.getCluster(), client.getClusterName()); + assertEquals(helper.getClusterName(), client.getClusterName()); client.checkMapping("test_smoke"); client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); client.flush(); @@ -54,8 +54,8 @@ class SmokeTest { int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkMetric().getFailed().getCount()); - assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); + assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java index dcd0e31..e394c9f 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java @@ -36,8 +36,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import java.util.concurrent.atomic.AtomicInteger; public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { @@ -47,20 +46,10 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - private Map nodes = new HashMap<>(); - - private Map clients = new HashMap<>(); - - private String home; - - private String cluster; - - private String host; - - private int port; - private static final String key = "es-instance"; + private static final AtomicInteger count = new AtomicInteger(0); + private static final ExtensionContext.Namespace ns = ExtensionContext.Namespace.create(TestExtension.class); @@ -73,20 +62,23 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - setHome(System.getProperty("path.home") + "/" + getRandomString(8)); - setClusterName("test-cluster-" + System.getProperty("user.name")); - return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create()); + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); } @Override - public void beforeEach(ExtensionContext context) throws Exception { - Helper helper = context.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); - logger.info("starting cluster"); + public void beforeEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); helper.startNode("1"); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); Object obj = response.iterator().next().getTransport().getAddress() .publishAddress(); + String host = null; + int port = 0; if (obj instanceof InetSocketTransportAddress) { InetSocketTransportAddress address = (InetSocketTransportAddress) obj; host = address.address().getHostName(); @@ -111,42 +103,26 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft } @Override - public void afterEach(ExtensionContext context) throws Exception { - closeNodes(); - deleteFiles(Paths.get(getHome() + "/data")); + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome() + "/data")); logger.info("data files wiped"); Thread.sleep(2000L); // let OS commit changes } - private void setClusterName(String cluster) { - this.cluster = cluster; - } - - private String getClusterName() { - return cluster; - } - - private void setHome(String home) { - this.home = home; - } - - private String getHome() { - return home; - } - - private void closeNodes() { + private void closeNodes(Helper helper) throws IOException { logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { + for (AbstractClient client : helper.clients.values()) { client.close(); } - clients.clear(); logger.info("closing all nodes"); - for (Node node : nodes.values()) { + for (Node node : helper.nodes.values()) { if (node != null) { node.close(); } } - nodes.clear(); logger.info("all nodes closed"); } @@ -168,20 +144,46 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft } } - private String getRandomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - private Helper create() { - return new Helper(); + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; } - class Helper { + static class Helper { + + String home; + + String cluster; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } + + Settings getNodeSettings() { + return Settings.builder() + .put("cluster.name", getClusterName()) + .put("path.home", getHome()) + .build(); + } void startNode(String id) { buildNode(id).start(); @@ -191,25 +193,24 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return clients.get(id); } - String getCluster() { - return getClusterName(); - } - - String randomString(int n) { - return getRandomString(n); + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); } private Node buildNode(String id) { - Settings nodeSettings = settingsBuilder() - .put("cluster.name", getClusterName()) - .put("path.home", getHome()) - .put("name", id) + Settings nodeSettings = Settings.builder() + .put(getNodeSettings()) + .put("node.name", id) .build(); Node node = new MockNode(nodeSettings); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); clients.put(id, client); - logger.info("clients={}", clients); return node; } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java index 73c0f81..409f38f 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java @@ -97,7 +97,7 @@ class ClientTest { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } @@ -126,7 +126,7 @@ class ClientTest { } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } @@ -183,7 +183,7 @@ class ClientTest { logger.warn("skipping, no node available"); } finally { client.stopBulk("test5", 60L, TimeUnit.SECONDS); - assertEquals(maxthreads * maxloop, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(maxthreads * maxloop, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index 64207e7..37149f6 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -64,7 +64,7 @@ class DuplicateIDTest { logger.warn("skipping, no node available"); } finally { client.close(); - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index be4a87b..74d8dee 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -38,7 +38,7 @@ class SmokeTest extends TestExtension { client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - assertEquals(helper.getCluster(), client.getClusterName()); + assertEquals(helper.getClusterName(), client.getClusterName()); client.checkMapping("test_smoke"); client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); client.flush(); @@ -56,8 +56,8 @@ class SmokeTest extends TestExtension { assertEquals(2, replica); client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkMetric().getFailed().getCount()); - assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); + assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); + assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java index e37ca8c..7867e42 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/TestExtension.java @@ -36,8 +36,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.Map; import java.util.Random; - -import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import java.util.concurrent.atomic.AtomicInteger; public class TestExtension implements ParameterResolver, BeforeEachCallback, AfterEachCallback { @@ -47,19 +46,9 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft private static final char[] numbersAndLetters = ("0123456789abcdefghijklmnopqrstuvwxyz").toCharArray(); - private Map nodes = new HashMap<>(); + private static final String key = "es-instance"; - private Map clients = new HashMap<>(); - - private String home; - - private String cluster; - - private String host; - - private int port; - - private static final String key = "es-test-instance"; + private static final AtomicInteger count = new AtomicInteger(0); private static final ExtensionContext.Namespace ns = ExtensionContext.Namespace.create(TestExtension.class); @@ -73,15 +62,16 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft @Override public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException { - setHome(System.getProperty("path.home") + "/" + getRandomString(8)); - setClusterName("test-cluster-" + System.getProperty("user.name")); - return extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); + // initialize new helper here, increase counter + return extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.incrementAndGet(), key -> create(), Helper.class); } @Override public void beforeEach(ExtensionContext extensionContext) throws Exception { - Helper helper = extensionContext.getParent().get().getStore(ns).getOrComputeIfAbsent(key, key -> create(), Helper.class); - logger.info("starting cluster"); + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + logger.info("starting cluster with helper " + helper + " at " + helper.getHome()); helper.startNode("1"); NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().transport(true); NodesInfoResponse response = helper.client("1"). execute(NodesInfoAction.INSTANCE, nodesInfoRequest).actionGet(); @@ -89,8 +79,8 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft .publishAddress(); if (obj instanceof InetSocketTransportAddress) { InetSocketTransportAddress address = (InetSocketTransportAddress) obj; - host = address.address().getHostName(); - port = address.address().getPort(); + helper.host = address.address().getHostName(); + helper.port = address.address().getPort(); } try { ClusterHealthResponse healthResponse = helper.client("1").execute(ClusterHealthAction.INSTANCE, @@ -107,46 +97,30 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft ClusterStateResponse clusterStateResponse = helper.client("1").execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); logger.info("cluster name = {}", clusterStateResponse.getClusterName().value()); - logger.info("host = {} port = {}", host, port); + logger.info("host = {} port = {}", helper.host, helper.port); } @Override - public void afterEach(ExtensionContext context) throws Exception { - closeNodes(); - deleteFiles(Paths.get(getHome() + "/data")); - logger.info("data files wiped: " + getHome()); + public void afterEach(ExtensionContext extensionContext) throws Exception { + Helper helper = extensionContext.getParent().get().getStore(ns) + .getOrComputeIfAbsent(key + count.get(), key -> create(), Helper.class); + closeNodes(helper); + deleteFiles(Paths.get(helper.getHome() + "/data")); + logger.info("data files wiped"); Thread.sleep(2000L); // let OS commit changes } - private void setClusterName(String cluster) { - this.cluster = cluster; - } - - private String getClusterName() { - return cluster; - } - - private void setHome(String home) { - this.home = home; - } - - private String getHome() { - return home; - } - - private void closeNodes() { + private void closeNodes(Helper helper) throws IOException { logger.info("closing all clients"); - for (AbstractClient client : clients.values()) { + for (AbstractClient client : helper.clients.values()) { client.close(); } - clients.clear(); logger.info("closing all nodes"); - for (Node node : nodes.values()) { + for (Node node : helper.nodes.values()) { if (node != null) { node.close(); } } - nodes.clear(); logger.info("all nodes closed"); } @@ -168,34 +142,57 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft } } - private String getRandomString(int len) { - final char[] buf = new char[len]; - final int n = numbersAndLetters.length - 1; - for (int i = 0; i < buf.length; i++) { - buf[i] = numbersAndLetters[random.nextInt(n)]; - } - return new String(buf); - } - private Helper create() { - return new Helper(); + Helper helper = new Helper(); + helper.setHome(System.getProperty("path.home") + "/" + helper.randomString(8)); + helper.setClusterName("test-cluster-" + helper.randomString(8)); + logger.info("cluster: " + helper.getClusterName() + " home: " + helper.getHome()); + return helper; } - class Helper { + static class Helper { + + String home; + + String cluster; + + String host; + + int port; + + Map nodes = new HashMap<>(); + + Map clients = new HashMap<>(); + + void setHome(String home) { + this.home = home; + } + + String getHome() { + return home; + } + + void setClusterName(String cluster) { + this.cluster = cluster; + } + + String getClusterName() { + return cluster; + } Settings getNodeSettings() { - return settingsBuilder() + return Settings.builder() .put("cluster.name", getClusterName()) .put("path.home", getHome()) .build(); } Settings getTransportSettings() { - return settingsBuilder() + return Settings.builder() + .put("cluster.name", cluster) + .put("path.home", getHome()) .put("host", host) .put("port", port) - .put("cluster.name", getClusterName()) - .put("path.home", getHome() + "/transport") .build(); } @@ -207,24 +204,24 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft return clients.get(id); } - String getCluster() { - return getClusterName(); - } - - String randomString(int n) { - return getRandomString(n); + String randomString(int len) { + final char[] buf = new char[len]; + final int n = numbersAndLetters.length - 1; + for (int i = 0; i < buf.length; i++) { + buf[i] = numbersAndLetters[random.nextInt(n)]; + } + return new String(buf); } private Node buildNode(String id) { - Settings nodeSettings = settingsBuilder() + Settings nodeSettings = Settings.builder() .put(getNodeSettings()) - .put("name", id) + .put("node.name", id) .build(); Node node = new MockNode(nodeSettings); AbstractClient client = (AbstractClient) node.client(); nodes.put(id, node); clients.put(id, client); - logger.info("clients={}", clients); return node; } } diff --git a/gradle.properties b/gradle.properties index 4b70d38..4ed757c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ group = org.xbib name = elx -version = 2.2.1.15 +version = 2.2.1.16 # main xbib-metrics.version = 2.0.0 @@ -8,7 +8,7 @@ xbib-metrics.version = 2.0.0 xbib-guice.version = 4.4.2 # guava 18 -> our guava 28.1 xbib-guava.version = 28.1 -xbib-netty-http.version = 4.1.48.0 +xbib-netty-http.version = 4.1.49.0 elasticsearch.version = 2.2.1 #jackson 2.6.7 (original ES version) -> 2.9.10 jackson.version = 2.9.10 @@ -17,8 +17,5 @@ log4j.version = 2.12.1 mustache.version = 0.9.5 jts.version = 1.13 -# test -junit.version = 5.5.1 - -# docs +junit.version = 5.6.2 asciidoclet.version = 1.5.4 From 246b8c10cd17cab4c8e20e5aaaf129d397b33673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Wed, 13 May 2020 20:44:47 +0200 Subject: [PATCH 18/19] cosmetic --- .gitignore | 2 ++ README.adoc | 2 +- .../main/java/org/xbib/elx/api/ExtendedClient.java | 14 ++++++++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index bdd8810..314748d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,7 @@ /.project /.gradle build +out +plugins *.iml *~ diff --git a/README.adoc b/README.adoc index 852224a..aa39fdb 100644 --- a/README.adoc +++ b/README.adoc @@ -1,4 +1,4 @@ -# Elasticsearch Extras - Client +# Elasticsearch Clients image:https://api.travis-ci.org/xbib/content.svg[title="Build status", link="https://travis-ci.org/jprante/elasticsearch-extras-client/"] image:https://img.shields.io/sonar/http/nemo.sonarqube.com/org.xbib%3Aelasticsearch-extras-client/coverage.svg?style=flat-square[title="Coverage", link="https://sonarqube.com/dashboard/index?id=org.xbib%3Aelasticsearch-extras-client"] diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java index 6114621..d6c761e 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java @@ -121,8 +121,9 @@ public interface ExtendedClient extends Flushable, Closeable { * @param id the id * @param source the source * @return this + * @throws IOException if update fails */ - ExtendedClient update(String index, String id, BytesReference source); + ExtendedClient update(String index, String id, BytesReference source) throws IOException; /** * Update document. Use with precaution! Does not work in all cases. @@ -170,7 +171,7 @@ public interface ExtendedClient extends Flushable, Closeable { * @param index index * @param settings settings * @return this - * @throws IOException if settings/mapping is invalid or index creation fails + * @throws IOException if settings is invalid or index creation fails */ ExtendedClient newIndex(String index, Settings settings) throws IOException; @@ -185,6 +186,15 @@ public interface ExtendedClient extends Flushable, Closeable { */ ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException; + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @return this + * @throws IOException if settings/mapping is invalid or index creation fails + */ ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; /** From 6a48751b901f0c65e3a6c19e5b10be074c27d80c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Fri, 22 May 2020 14:04:42 +0200 Subject: [PATCH 19/19] update to Gradle 6.4.1 --- build.gradle | 215 +++-------------------- elx-api/build.gradle | 14 +- elx-common/build.gradle | 10 +- elx-http/build.gradle | 4 +- elx-node/build.gradle | 4 +- elx-transport/build.gradle | 2 +- gradle.properties | 10 +- gradle/compile/java.gradle | 35 ++++ gradle/documentation/asciidoc.gradle | 55 ++++++ gradle/ide/idea.gradle | 13 ++ gradle/publish.gradle | 0 gradle/publishing/publication.gradle | 64 +++++++ gradle/publishing/sonatype.gradle | 11 ++ gradle/test/junit5.gradle | 35 ++++ gradle/wrapper/gradle-wrapper.jar | Bin 55616 -> 58910 bytes gradle/wrapper/gradle-wrapper.properties | 5 +- gradlew | 31 ++-- gradlew.bat | 4 + 18 files changed, 278 insertions(+), 234 deletions(-) create mode 100644 gradle/compile/java.gradle create mode 100644 gradle/documentation/asciidoc.gradle create mode 100644 gradle/ide/idea.gradle delete mode 100644 gradle/publish.gradle create mode 100644 gradle/publishing/publication.gradle create mode 100644 gradle/publishing/sonatype.gradle create mode 100644 gradle/test/junit5.gradle diff --git a/build.gradle b/build.gradle index 60e581e..3f71fc8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,206 +1,41 @@ plugins { - id "org.sonarqube" version "2.8" + id "de.marcphilipp.nexus-publish" version "0.4.0" id "io.codearte.nexus-staging" version "0.21.1" - id "com.github.spotbugs" version "2.0.1" - id "org.xbib.gradle.plugin.asciidoctor" version "1.6.0.1" } -if (JavaVersion.current() < JavaVersion.VERSION_11) { - throw new GradleException("This build must be run with Java/OpenJDK 11+") +wrapper { + gradleVersion = "${project.property('gradle.wrapper.version')}" + distributionType = Wrapper.DistributionType.ALL +} + +ext { + user = 'jprante' + name = 'elx' + description = 'Extensions for Elasticsearch clients (node and transport)' + inceptionYear = '2019' + url = 'https://github.com/' + user + '/' + name + scmUrl = 'https://github.com/' + user + '/' + name + scmConnection = 'scm:git:git://github.com/' + user + '/' + name + '.git' + scmDeveloperConnection = 'scm:git:ssh://git@github.com:' + user + '/' + name + '.git' + issueManagementSystem = 'Github' + issueManagementUrl = ext.scmUrl + '/issues' + licenseName = 'The Apache License, Version 2.0' + licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt' } subprojects { - apply plugin: 'java' - apply plugin: 'maven' - apply plugin: 'pmd' - apply plugin: 'checkstyle' - apply plugin: 'com.github.spotbugs' - apply plugin: 'org.xbib.gradle.plugin.asciidoctor' - - configurations { - asciidoclet - } + apply plugin: 'java-library' dependencies { - testImplementation "org.junit.jupiter:junit-jupiter-api:${project.property('junit.version')}" testImplementation "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" testImplementation "org.apache.logging.log4j:log4j-jul:${project.property('log4j.version')}" testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${project.property('log4j.version')}" - testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${project.property('junit.version')}" - asciidoclet "org.xbib:asciidoclet:${project.property('asciidoclet.version')}" } - compileJava { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 - } - - compileTestJava { - sourceCompatibility = JavaVersion.VERSION_11 - targetCompatibility = JavaVersion.VERSION_11 - } - - tasks.withType(JavaCompile) { - options.compilerArgs << "-Xlint:all" - if (!options.compilerArgs.contains("-processor")) { - options.compilerArgs << '-proc:none' - } - } - - test { - enabled = true - useJUnitPlatform() - jvmArgs = [ - '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', - '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', - '--add-opens=java.base/java.nio=ALL-UNNAMED' - ] - systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' - systemProperty 'jna.debug_load', 'true' - systemProperty 'path.home', "${project.buildDir}" - failFast = false - testLogging { - events 'PASSED', 'FAILED', 'SKIPPED' - } - afterSuite { desc, result -> - if (!desc.parent) { - println "\nTest result: ${result.resultType}" - println "Test summary: ${result.testCount} tests, " + - "${result.successfulTestCount} succeeded, " + - "${result.failedTestCount} failed, " + - "${result.skippedTestCount} skipped" - } - } - } - - clean { - delete "out" - } - - task javadocJar(type: Jar, dependsOn: javadoc) { - archiveClassifier.set('javadoc') - } - - task sourcesJar(type: Jar, dependsOn: classes) { - from sourceSets.main.allSource - archiveClassifier.set('sources') - } - - artifacts { - archives javadocJar, sourcesJar - } - - spotbugs { - effort = "max" - reportLevel = "low" - //includeFilter = file("findbugs-exclude.xml") - } - - tasks.withType(com.github.spotbugs.SpotBugsTask) { - ignoreFailures = true - reports { - xml.enabled = false - html.enabled = true - } - } - - tasks.withType(Pmd) { - ignoreFailures = true - reports { - xml.enabled = true - html.enabled = true - } - } - - tasks.withType(Checkstyle) { - ignoreFailures = true - reports { - xml.enabled = true - html.enabled = true - } - } - - pmd { - toolVersion = '6.11.0' - ruleSets = ['category/java/bestpractices.xml'] - } - - checkstyle { - configFile = rootProject.file('config/checkstyle/checkstyle.xml') - ignoreFailures = true - showViolations = true - } - - sonarqube { - properties { - property "sonar.projectName", "${project.group} ${project.name}" - property "sonar.sourceEncoding", "UTF-8" - property "sonar.tests", "src/test/java" - property "sonar.scm.provider", "git" - property "sonar.junit.reportsPath", "build/test-results/test/" - } - } - - ext { - description = 'Extensions for Elasticsearch clients (node and transport)' - scmUrl = 'https://github.com/jprante/elx' - scmConnection = 'scm:git:git://github.com/jprante/elx.git' - scmDeveloperConnection = 'scm:git:git://github.com/jprante/elx.git' - } - - task sonaTypeUpload(type: Upload, dependsOn: build) { - group = 'publish' - configuration = configurations.archives - uploadDescriptor = true - repositories { - if (project.hasProperty('ossrhUsername')) { - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: 'https://oss.sonatype.org/service/local/staging/deploy/maven2') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - snapshotRepository(url: 'https://oss.sonatype.org/content/repositories/snapshots') { - authentication(userName: ossrhUsername, password: ossrhPassword) - } - pom.project { - groupId project.group - artifactId project.name - version project.version - name project.name - description description - packaging 'jar' - inceptionYear '2019' - url scmUrl - organization { - name 'xbib' - url 'http://xbib.org' - } - developers { - developer { - id 'xbib' - name 'Jörg Prante' - email 'joergprante@gmail.com' - url 'https://github.com/jprante' - } - } - scm { - url scmUrl - connection scmConnection - developerConnection scmDeveloperConnection - } - licenses { - license { - name 'The Apache License, Version 2.0' - url 'http://www.apache.org/licenses/LICENSE-2.0.txt' - } - } - } - } - } - } - } + apply from: rootProject.file('gradle/ide/idea.gradle') + apply from: rootProject.file('gradle/compile/java.gradle') + apply from: rootProject.file('gradle/test/junit5.gradle') + apply from: rootProject.file('gradle/publishing/publication.gradle') } -nexusStaging { - packageGroup = "org.xbib" -} +apply from: rootProject.file('gradle/publishing/sonatype.gradle') diff --git a/elx-api/build.gradle b/elx-api/build.gradle index 41bb17d..ac3cd45 100644 --- a/elx-api/build.gradle +++ b/elx-api/build.gradle @@ -1,6 +1,6 @@ dependencies { - compile "org.xbib:metrics-common:${project.property('xbib-metrics.version')}" - compile("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") { + api "org.xbib:metrics-common:${project.property('xbib-metrics.version')}" + api("org.elasticsearch:elasticsearch:${project.property('elasticsearch.version')}") { // exclude original ES jackson yaml, cbor, smile version (2.6.2) exclude group: 'com.fasterxml.jackson.dataformat' // these dependencies that are not meant for client applications @@ -11,14 +11,14 @@ dependencies { exclude group: 'com.google.guava' } // override log4j2 of Elastic with ours - compile "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" + api "org.apache.logging.log4j:log4j-core:${project.property('log4j.version')}" // override ES jackson with our jackson version // for Elasticsearch session, ES uses SMILE when encoding source for SearchRequest - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson.version')}" + api "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${project.property('jackson.version')}" // CBOR ist default JSON content compression encoding in ES 2.2.1 - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson.version')}" + api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${project.property('jackson.version')}" // not used, but maybe in other projects - compile "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson.version')}" + api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${project.property('jackson.version')}" // lift guava - compile "org.xbib:guava:${project.property('xbib-guava.version')}" + api "org.xbib:guava:${project.property('xbib-guava.version')}" } \ No newline at end of file diff --git a/elx-common/build.gradle b/elx-common/build.gradle index 25b9d83..a89f8f3 100644 --- a/elx-common/build.gradle +++ b/elx-common/build.gradle @@ -1,7 +1,7 @@ dependencies { - compile project(':elx-api') - compile "org.xbib:guice:${project.property('xbib-guice.version')}" - runtime "com.vividsolutions:jts:${project.property('jts.version')}" - runtime "com.github.spullara.mustache.java:compiler:${project.property('mustache.version')}" - runtime "net.java.dev.jna:jna:${project.property('jna.version')}" + api project(':elx-api') + implementation "org.xbib:guice:${project.property('xbib-guice.version')}" + runtimeOnly "com.vividsolutions:jts:${project.property('jts.version')}" + runtimeOnly "com.github.spullara.mustache.java:compiler:${project.property('mustache.version')}" + runtimeOnly "net.java.dev.jna:jna:${project.property('jna.version')}" } diff --git a/elx-http/build.gradle b/elx-http/build.gradle index 021d75d..25d88f4 100644 --- a/elx-http/build.gradle +++ b/elx-http/build.gradle @@ -1,4 +1,4 @@ dependencies { - compile project(':elx-common') - compile "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" + api project(':elx-common') + api "org.xbib:netty-http-client:${project.property('xbib-netty-http.version')}" } \ No newline at end of file diff --git a/elx-node/build.gradle b/elx-node/build.gradle index bc5e01e..7752beb 100644 --- a/elx-node/build.gradle +++ b/elx-node/build.gradle @@ -1,3 +1,3 @@ dependencies { - compile project(':elx-common') -} \ No newline at end of file + api project(':elx-common') +} diff --git a/elx-transport/build.gradle b/elx-transport/build.gradle index bc5e01e..6c132b1 100644 --- a/elx-transport/build.gradle +++ b/elx-transport/build.gradle @@ -1,3 +1,3 @@ dependencies { - compile project(':elx-common') + api project(':elx-common') } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 4ed757c..04cf885 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,21 +1,17 @@ group = org.xbib name = elx -version = 2.2.1.16 +version = 2.2.1.17 -# main -xbib-metrics.version = 2.0.0 -# guice 4 - > our guice 4.4.2 +gradle.wrapper.version = 6.4.1 +xbib-metrics.version = 2.1.0 xbib-guice.version = 4.4.2 -# guava 18 -> our guava 28.1 xbib-guava.version = 28.1 xbib-netty-http.version = 4.1.49.0 elasticsearch.version = 2.2.1 -#jackson 2.6.7 (original ES version) -> 2.9.10 jackson.version = 2.9.10 jna.version = 4.5.2 log4j.version = 2.12.1 mustache.version = 0.9.5 jts.version = 1.13 - junit.version = 5.6.2 asciidoclet.version = 1.5.4 diff --git a/gradle/compile/java.gradle b/gradle/compile/java.gradle new file mode 100644 index 0000000..a9d76ce --- /dev/null +++ b/gradle/compile/java.gradle @@ -0,0 +1,35 @@ + +apply plugin: 'java-library' + +java { + modularity.inferModulePath.set(true) +} + +compileJava { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +compileTestJava { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +jar { + manifest { + attributes('Implementation-Version': project.version) + } +} + +task sourcesJar(type: Jar, dependsOn: classes) { + classifier 'sources' + from sourceSets.main.allSource +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + classifier 'javadoc' +} + +artifacts { + archives sourcesJar, javadocJar +} \ No newline at end of file diff --git a/gradle/documentation/asciidoc.gradle b/gradle/documentation/asciidoc.gradle new file mode 100644 index 0000000..87ba22e --- /dev/null +++ b/gradle/documentation/asciidoc.gradle @@ -0,0 +1,55 @@ +apply plugin: 'org.xbib.gradle.plugin.asciidoctor' + +configurations { + asciidoclet +} + +dependencies { + asciidoclet "org.asciidoctor:asciidoclet:${project.property('asciidoclet.version')}" +} + + +asciidoctor { + backends 'html5' + outputDir = file("${rootProject.projectDir}/docs") + separateOutputDirs = false + attributes 'source-highlighter': 'coderay', + idprefix: '', + idseparator: '-', + toc: 'left', + doctype: 'book', + icons: 'font', + encoding: 'utf-8', + sectlink: true, + sectanchors: true, + linkattrs: true, + imagesdir: 'img', + stylesheet: "${projectDir}/src/docs/asciidoc/css/foundation.css" +} + + +/*javadoc { +options.docletpath = configurations.asciidoclet.files.asType(List) +options.doclet = 'org.asciidoctor.Asciidoclet' +//options.overview = "src/docs/asciidoclet/overview.adoc" +options.addStringOption "-base-dir", "${projectDir}" +options.addStringOption "-attribute", + "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" +configure(options) { + noTimestamp = true +} +}*/ + + +/*javadoc { + options.docletpath = configurations.asciidoclet.files.asType(List) + options.doclet = 'org.asciidoctor.Asciidoclet' + options.overview = "${rootProject.projectDir}/src/docs/asciidoclet/overview.adoc" + options.addStringOption "-base-dir", "${projectDir}" + options.addStringOption "-attribute", + "name=${project.name},version=${project.version},title-link=https://github.com/xbib/${project.name}" + options.destinationDirectory(file("${projectDir}/docs/javadoc")) + configure(options) { + noTimestamp = true + } +}*/ diff --git a/gradle/ide/idea.gradle b/gradle/ide/idea.gradle new file mode 100644 index 0000000..64e2167 --- /dev/null +++ b/gradle/ide/idea.gradle @@ -0,0 +1,13 @@ +apply plugin: 'idea' + +idea { + module { + outputDir file('build/classes/java/main') + testOutputDir file('build/classes/java/test') + } +} + +if (project.convention.findPlugin(JavaPluginConvention)) { + //sourceSets.main.output.classesDirs = file("build/classes/java/main") + //sourceSets.test.output.classesDirs = file("build/classes/java/test") +} diff --git a/gradle/publish.gradle b/gradle/publish.gradle deleted file mode 100644 index e69de29..0000000 diff --git a/gradle/publishing/publication.gradle b/gradle/publishing/publication.gradle new file mode 100644 index 0000000..c35fcb9 --- /dev/null +++ b/gradle/publishing/publication.gradle @@ -0,0 +1,64 @@ + +apply plugin: "de.marcphilipp.nexus-publish" + +publishing { + publications { + mavenJava(MavenPublication) { + from components.java + artifact sourcesJar + artifact javadocJar + pom { + name = project.name + description = rootProject.ext.description + url = rootProject.ext.url + inceptionYear = rootProject.ext.inceptionYear + packaging = 'jar' + organization { + name = 'xbib' + url = 'https://xbib.org' + } + developers { + developer { + id = 'jprante' + name = 'Jörg Prante' + email = 'joergprante@gmail.com' + url = 'https://github.com/jprante' + } + } + scm { + url = rootProject.ext.scmUrl + connection = rootProject.ext.scmConnection + developerConnection = rootProject.ext.scmDeveloperConnection + } + issueManagement { + system = rootProject.ext.issueManagementSystem + url = rootProject.ext.issueManagementUrl + } + licenses { + license { + name = rootProject.ext.licenseName + url = rootProject.ext.licenseUrl + distribution = 'repo' + } + } + } + } + } +} + +if (project.hasProperty("signing.keyId")) { + apply plugin: 'signing' + signing { + sign publishing.publications.mavenJava + } +} + +nexusPublishing { + repositories { + sonatype { + username = project.property('ossrhUsername') + password = project.property('ossrhPassword') + packageGroup = "org.xbib" + } + } +} diff --git a/gradle/publishing/sonatype.gradle b/gradle/publishing/sonatype.gradle new file mode 100644 index 0000000..e1813f3 --- /dev/null +++ b/gradle/publishing/sonatype.gradle @@ -0,0 +1,11 @@ + +if (project.hasProperty('ossrhUsername') && project.hasProperty('ossrhPassword')) { + + apply plugin: 'io.codearte.nexus-staging' + + nexusStaging { + username = project.property('ossrhUsername') + password = project.property('ossrhPassword') + packageGroup = "org.xbib" + } +} diff --git a/gradle/test/junit5.gradle b/gradle/test/junit5.gradle new file mode 100644 index 0000000..b02dea8 --- /dev/null +++ b/gradle/test/junit5.gradle @@ -0,0 +1,35 @@ + +def junitVersion = project.hasProperty('junit.version')?project.property('junit.version'):'5.6.2' +def hamcrestVersion = project.hasProperty('hamcrest.version')?project.property('hamcrest.version'):'2.2' + +dependencies { + testImplementation "org.junit.jupiter:junit-jupiter-api:${junitVersion}" + testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" + testImplementation "org.hamcrest:hamcrest-library:${hamcrestVersion}" + testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}" +} + +test { + useJUnitPlatform() + failFast = true + jvmArgs = [ + '--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED', + '--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED', + '--add-opens=java.base/java.nio=ALL-UNNAMED' + ] + systemProperty 'java.util.logging.manager', 'org.apache.logging.log4j.jul.LogManager' + systemProperty 'jna.debug_load', 'true' + systemProperty 'path.home', "${project.buildDir}" + testLogging { + events 'STARTED', 'PASSED', 'FAILED', 'SKIPPED' + } + afterSuite { desc, result -> + if (!desc.parent) { + println "\nTest result: ${result.resultType}" + println "Test summary: ${result.testCount} tests, " + + "${result.successfulTestCount} succeeded, " + + "${result.failedTestCount} failed, " + + "${result.skippedTestCount} skipped" + } + } +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 5c2d1cf016b3885f6930543d57b744ea8c220a1a..62d4c053550b91381bbd28b1afc82d634bf73a8a 100644 GIT binary patch delta 23020 zcmZ6yV{|24)TJ9%DzUY6ksRrv&p?~YSFgp`E zPcOBdd*dQ@-0EJu5{@Rs)8Fukc%2C>|tqZrK zLV^5dGp39tCCDe?lhegZW_)s9c6qS=x5>=o1QrccXaggTb{|SR<7_H(l(Z70W`&rZ z*Dxcys#XiG^E_s)xh_o-)aOX`Ma z22{yd+*PXI5-7hr-c~6pL}6@nDGtj_>wy=jiKd=H%jHgid{1{wgQPaWxAjzYe*kAp zi6)6~N&I~4K!uX0dVFK3POvLlP6oxF!bjU1Ve+ir^&Pon_&F5%I`^Rto4r5|03Z6R zHcKNn>+3Jh5>TT$JJ&1ezdWQ|XW&Z(i|Ngc8*n^8ueC1TEDZtJ52X`>_+<<$x zTag^mA={k`be3UGne|-d9(u01VN=NDn${}U;#5h@UYugK7(Cei>D!F3NYDjm0 zr#f_FwHMwCR7roR&t6dm90CReUHYTRI`xupteox3}eJqc5Y|pg|k*S)!lqA{;l88&*ZBt z%?qh+CBVdkN8U~;DdVo*$+gCvrx!iWpFdw;%zlNhQsN+}EcyeG(3m1=NlfG>I5*?B zLdi^|M(W~lsiwNa-Ymy)K=hHWC?pJ1-99zPE<#~>#hPFTy?!VZWfqb)ViP=0a-Y?3 zeIGI*SR-qEM-DXtCis`Q;{4C~r`z}>UXmX?P8t4Hn$ zr*VYtNz0COSGJA4XD*REx*9m}@QZW$tC$oHw#Kw+o)$Zd5&j6fV#d>7gzk5iZt;AV zHl&k8-VBu0C`M;?>gK1nZ3`K4rw=J-w`wEsI?YXBl*Wq-OznyKX)3sy!tJ7EdME=i zzjhd;+Auz`ujzCkfgzfKep9J%IqvfWv1IO?ly~;PV~SDqjxd0aCkx2Op@$B+S$gki zUGwUQ?2`bga4U_8B&Iw=kU$vu1N`xRbuMT0g*ll|;3^y{5WCVCZNKDb5Pm##;9#01 z*1&}pt=1$`AvZhn!aQ4dcC55V6B%P)B=9I7O=Kn|Q6{qWcXiMP3|tZf%6;PrH+eA# zE;kHamE^0YKm2%kVO{_)%yonbNF~ZLnoocN`sTpjafW91@rx2weCu&=YLtRqRv-Cl z8tnk(4P`Y*puw&Tq`{u`d6CyVYlzx$K@Ut%pCP_(H4a;(Djevc4&=_Bf56S1U&$qU zsLgIz0O2j^z$`5d)8n67AmF}56aMHDkS(rFCX=E!Z`q(gk(0eNeD`#&Za;S%zP%)_ zQRVMvLO&@Nza!eBzQ{`oR6dkKde(;hc)1h>+=8qdU zn>wZ#%B-?)=qNq04ClhR=lEyi9F4M>T)Wz@_{BeexLgQxYj$nAPFJtF4|&Z4*Rj&F zYS_8hvSvflEmTtL}z{^AQ0NI~`qDUdgu3gZ{$cf7F z--E9+IBS3xarvcZj0Vkvt^gthw~%NAQAj2>ehESL4s{*-r@v)QB{Ch8{No`$_$k}LXnRlc-uQSf%^ z>4RFo>ELnm>$#j%?XlX#kKpU#6nAa#d-K=68eSUq7p`7m?AJd*oG1LhqXZqr=zuU} zEJdzt)pDvOt2n_@=x1gy8sV74O`;A*%3l`Ld`>67U=j#RKoC)}K=~k^Ax3_4jhSxt za4DRDo9^GZs|5dmBwGKvgC{CwF6D&)0a=Cmub=+!4hn>Vh7vV1H*&RcNxZ-!02Zrl zso<)i{cPEAz)t*9mZVB*@oQm+kgiZ~Dyp{ix3r{6Rz#}iNj>DS>Ac)G2miu--&HYq zFHn#-@Hvb6y_EheXTjs!eHt}=`no&v%6tIa^|&VYae?8&PaJD^pJ_k&^gS`LC-{80 z;|H19rAKw_QjEpOiZfP>h?3ej1>(!6b46p~#`R*Cb6fA@h2StnA%kmyGp2{_AC0DC zm&JzTi5GL(8&38@f*-&kkI<$D&&ZW*D^m|SZqvaa4@oiorZG4KXh59Fptet*mQ9ZB z7AL^<&c4>A6Kg-}!r&YR19cFN%Nh#dj=dbPnWcXZS)2Ier#K2bxvuE+O%&{!ag&!( zuGDEqs_jrt6!IrdTi>>%PDjV;GDZaVKI4`PGYYI)$N(s%h!{Mt4h{fwl7(@Mp{uLEbX1mMO>4?RT( z+i{y<^N8FmkGP{<1iqQtRUWK^Iz`xHMsZPux(xz0q-~w1Us95UN|xIISO-p2O`g7b z(~1m~7Lu8*<+k{pi*R}_xWhs|`T1J%aC*QgCreBCvG#m++2=1V!FwY9sRaE(_8e_c znI{I)q;s8+l=&3op2993sTN_%H|{rIe=nbKaUU^B15|=jVCm;vw8(i!G_7ul8w$6m zcg7UX_D9$(Ce~%ezc+^GqCY{;*psRn)*g^NW0|c;OfTq2%@n%ca(Z+yrs5=mOSzmITuiKqj|M>!WA#RZr@MR5`g>V`6${VovvXDaCtOdnz@whTFNWQ7#o3e4WZ&)>ewjw7*CTlQ!hQZtu}1aDUzdn9M66cvYgE8d0A-yJfo7v^rfxTyu1lLkPsf`)q#f1| zZo1^M449#bD0`Ehs5e8tffNeEUn+mkPMWm;8rZbeZZhZHOkCO-^OS5w z+|jYNVWhT{J%QEq_rZftlzfByzlD-SdNVnP2m-=G1Oh_*fBvT5!2g-ys!HTqv;v_J!j0=Z9L#_GJC2j#RLJ^$^S$Yb7uH}K%^Ss8+NQs!5az9+rp??D-$ zG_Yj$rK@Nz*ZYj|l(G8}@J>7DV{rS~Z23Jb!+U0r{>~fhyIVZ;_@b{Z`m7Vc8*aW3k`edfY+`%N$n z6CA1O191+QXln1~S3Ii6FVOTEC7)>Q72AdKWnehH+QT!|?2uWs(R)<(qG{}v-x!iP zE4##ws{4m+g+JVq-ODyhoI1yO)p~S}^l1Kx(RE4eZNa!R0s2Jy>v#^lXv~Vz)UTPv zpIkP#O>(3L<@nd_(@j$1rCmpqTR_Dr2qi6MfLHPx2jcxPVlEpH_2y+ShW0yq+o%J2 z;Q59IjH?|A=cP}}Z8=!h@N4RH%*)fNO8^SxmYllKCy#Z-_QdiA6U&?J8h_OM-FOBH zgpRQD@bV5|p8bxX29-;Jvw&%*=%AsVg@L`j#t9Ky78cyI%M(0&&MCB;JCulC+8Sy< zaQ~@U_S?T^%Uha-$#egBHEng|-fjWM8o%+;a?_IEs zo)(=IRknGI``eO%{6zga&~#>6lf?YS_`EE4B%pmeQ)Z26^6;9ikg1`BH-*V-0hKg_ zy)1SRehkrlB*ZzO>%6d#Wv z$J1LHvW^GQvmg~$mNj;?mA7Cr!c`ZJH4)-5)9f69qpU>Ui+?;VlZ75K1WlHM|)@^B}YcmMl})1Yq+Tn*C3ut3qdyWTt@m9 zX1y*xETXWGh1UVZt#48_=a^YVi>ii6C3tX49YX8?J{>@79A*qDdKgzGHad3-#!sD5 z%H|ccT{Gv|!0we~sPzHlqiQ*ZpB9un48-XU+%BIJ^MXOfYP~X*Q6@@V?dspcd6{)B zNqK9Um(f=U z6AR#*W{y^X=@jTIMcPg4Wv0z;iZ1JSjHbl8(@6_W@9|+dIGhIj&YDy44+y63zYOXx z8J6-DG>HTqE@G;Za~3{WAVHwr0RdA&ZfvY4#gClO){0wzGd5ggp#;Op-D@o14pl~s zI(IN{qBpcKv-$)Qb<@~*(VlnInX~!gQoIt+@zHhNhn;rov7I#15(L|y^zr$PM#Y13 zf#=m3I_>ELBzrl)rM9H8LGu%PyR#Um!!2jU_&ypgzD+ zgusW8aF`Tu)ddxL*QiMYz`2+L6NII%lVA*$40B~e1<`=PhC|}>0j1=#gq(i4##%JL zG=9x{2Ipg_7Aab2*F)j=KAT3*p5o}Oyn+do+#P%}cxc*mK$0BL_^xiHytuy5Cczf#> zpv36N^X^u0lR~*zHw=VhM;MoAH(2;%c9S$P?0Ev;)>SXoRr7ppEA+lM6la%SJ$(sx zt03kfUJh^EXph0?qT9YU(`kjE?j5d^I`#}diW#AT4U?!08LKX?3vLl(bdJ+6$z2@B zNK?*se>U*7b@gBx@9<9RYEXhf_34^t#uB~tIi;vsKXRYmkL9pioKK03AyfTG{!kZ6 zmn{M5ks51irtt4rG0kTN%Pl9R$E-h)lag(ve~!4FIVI4hkN%zJG5NfCo+-BTm5CUImX-|iSR1uMwhS^V1Ke%-LR<6XIVdsX8Frm;=t%6bCR z;$kWVcSYhIMf{2bzI|EEEWE!!97;PB$qYAeMriJeG>y84o9a8< z1~3s-X`a>(Q5nlYb~6nXgU9yin^!JS)p<UCTyZdhr^8p`SHNa49f6Y zN#>h`C^JhPsbbzU=trD}s7E`m*v|?bV)Xh;Xf?A2%B1$92ughP8NuLJgsvK+gmE04 z+JwsdH=1_b&+AInZ6>iC2a^RdY7Hq!k)f#X4JqW@-ux$q(Jk4freKUYbuN3{>f)>U zk}0h*&&=f`TW!i#9SZN0VmV*|WtDSa1g6wT#~%Rw97*mAj@ow7IrV%$pOo54nfB^_ zLW}Ce)79;eytZU#ksW>K^xRj|;ZZQrACVbhvE@#N_PP>k5q9hf{!OBm4ny-;t@cjh zR`4%*YR)*lgdG+5?$n&b4y#j7FGBSm!oQqUBXd83^63|Bo`>!m9M{03miuAr!-?5` zju|w17=^igCbSi~`2_uclp-=4oAA-TFJy0f_hz;cJ6Af_8Z0iEJKctVZA0wbb7;S= z;UL$?F&7swmpIg-u2;{iSJk!ZtH%wlHCUTyx|$0vO-ywQm$0=Th_yetwJ&s1^Pf~y z37>7#ib8N*lOt&vPB(xw7Zrz%6*X4K?DDF7Kgjb;N8Pud zR?mu!2Cnoqqly!-3o}OJN7hrb+28k>Meh@=P4#aSPwWslV5lH*B?T(N+wIw&R)}vf}YluLBf5Q_;teH@R#ZQ(m zeFs9O+n^4vqzc|~vE7S7y6d7}qX{BmhamVxNb5pq?@U_?q13`J^Hky-_(JIfV!@lC zRowo+w-pErk>B~aN~ol3=do@j<}O@NmIpoI9+TR@p;{6&*_VwxaisL9nsO>(;c1IbQ41N077J^fPBjbH%{6FGfM%c z3)NRycM%I*I3WYFDxYve1@H}a1T2~DTPFqzGje17s|4kFe&GNHZff}SiLWN*gQrR+ zia8>Q%^hA$t0^L$mw=uK*|+$Dz_S{rK_0`-K4EQsaM5`Nq!96g`F(bRRa_1%F^-sWU0OZjPtE={a`W)7{mVGZ3ADA6I@y`HPK=7aH5x^uXn1 zH>)v=3$p5i&L2;T`Owot^%YTxX#7a~SXrm!(H1{mlD|#9fCc%;t^__(8qa^t2Mr-L zlu)}MN14cIL>ACK#~FayFoG$xvEh=a4#o{k<$Viyu+^b&AY0FxN}U*GIB@!7a(>AC zMFRGUVgtlxf6?kaN)C`+OKbHloHXBTG`!-X5OKqyL^t8vK|R;?!j8;XuT=&NcxZu4;{g*RWtL8|G8FNjh67FieiHf=Jy3Y zqWz;pQ^ytA(^f#mDdu*GreK)q$k7eX#@6Ht|5#$*JgO;#(eEQ>CVy!Wu#RiwL`3j6 zNR6LE3fqbEe%MQUm22k|sCaHYd?NFsdB^ZNifcso1DKd$f5j|*cKQMOvuDIgK672s zooOAo!hUWIgfKjTF#p#XNPHYaZ(PL+LWk$JLEBgVj{}CFfPj$vk3GPE1l08OL32m{ z`ED4YO9Y))6vnj7)V6VE+@lE^5Ef>(MIj;%f+)(qM93s=WWBhg^Y>D;t(NG%NcPIr z6f0h}5uO|V5VY>LpHr`v;J+YzRuTJaZ_c^4MMuq?JM%s5{Ch; zPp2p;&EnX?&`io)dpBI4qiH5%@ue&f zFLWFMVM2dt*)vD)@qrpL$!&q`E+j)5z}9V*l2~O~BJv@H0u(DRC~4tpi@1(&MgYti zLFc-dDb(tcS#qzv6_Lrhp^Q=Ey&1z{3Kb5iNN>2?3E(UxtYJ++FA*%;#QOA<1L{>X z*^O~ha7c$i`?aO{i<;3*biOcV z$PLHD0J+sFLs-t4foPZ(MOkr*THLA(#akUr;03 z3Wc?1pM^&cc&^$l69u|YOyQ`>x{9QiZ@?8xN z*}j8UU~|Au-S-ZOd>UW@Zpz55pU`u&AvQjb@|G9+pT;fJr&3?yYhr-F0i&ocSXxk- zI<4(sh1^8Bd}iy#n3Z&%)kuLn8+1?yJG)srJ=s&O&+fmy8CQ%@o09#IV6Bvs znxd~uozG!qU2$++(;JeI++rG?rIhQ#u1j_@!fuf|Uiy45Sm2qOF&ok7@XlelMv)vp zoPlEIKCFy!Jc#8)XD_w8o}p~^ugy30qPzufR@b@Q4)Q6<;zYQ?`ND9VWj>}w#x-OV z$ZW+{QG8mUEv2F6Gp(Vx$9$}q-e{GnWjkPF1=|j29(tK$LM>5P zxmmjA_6=DdDStfs#>O6R@np5!sR!@|`Pc!uS!dDjNJ4CY|Ln6}o7-&4jE5o9z}DyK71 zVw=$kxZ)1A>5|OObkn5bVDIV}H3{zBBMPE=q277Xd~BeTp705)@pfyBOtbhyApEYn zS6MN(GlMQ#5FfIrn(CHhmAT{6nFY#O2O90nY?$Zr#-N7Z?UnFDFz!7+6|~|7Y%zNw zUrmU8m7(_JhQ!tE8~*vgzKSRScTJ#UmxosWqgQl%F4jZZMO!L^yGO3NkSY&5`UirR z=9L;k@gModLg47tncU&?Bb}4C7v~&jL98$Dt48};^sV_{Rk-%M`;D8bBE2|ZCN zYk3#?rKMyHreQ8cTT2o!3|B)@09VmHWv%J63j8&5M|}BGq_>gU=mAEny}5M)(*)7) z&)Oz$h?eQ;(w%o=Rz|K_R1}gF6zC#5lTlUR-~iVAm-3r8JsEE_k0H=&SwdjlX4Edp z(+XE%w@$RNa%5YuvN z`shlEx{&?)mYM7?M{$DdH8z@?xY_Mp<{(|p8_ zzihZE5--=zT@LUO>bCRs*|ER^j#Qxx*fbCK2KFrVTk6#J7wqwlq#L(z|1Kn53Q{tX_pjsk zEuh;5oYl{Q--Q2Hjr{Kv(kfg6nGe{1X|rD-AfW#fLjp4V21?mEyBOKnh*+ALSpN^S zNKVpIKovp@*#Wfbb!-XK`w|uS$mr1Mz+@l|VM#?%h%qwB-P&n}G}L{K+Esp&+PBS$ z7R(LGFn%fxvo^};NW-(q%}lOkzi{8IrEduUfuCTC{iNWIhi01cid3Vwf+e&(5l*`7 zae`7KIKV)qr`H}@eGaPtqb8rh9p?#vCL^8(UkNPKrsGD=Dn`#tWfKW%m4j!u)m0N; zseZAVS6f7%kJ^jQrfVxg@ZYNWj@uU9>yLhm)k%HTlX3W3D@ckeL;`z`V2j|SV11mW zxfYxcRnw}=#i0u~TLlmMx135{cAIPUwv$o5E@0WpwCsiNMhmsk>S)f5L)!TrrTi9v z-z@DW5YDnf*jy(RK@G1YVDA-uj~gmx)gsK?`36gwm)?%h{=s}TsZWg(wsMF>gxK4x z%~b3UMyFo_gbf_%sQ`{e9y|0p9*)a51o_Om@?TdLv-wp(6Ti-A!xnf82{DD;eoDBK zE|ArT#H?R%eh}^YT+>3cqkt3@wT*s2B*bN9BeR>-s1=LiCsrv_wGXaa3EJz9WKyuOL!WBMRp${&m3iX zQI-QYpOF8pZqga&s9W`$wrOX!Sz)$-(uH{az_otFng(L05C|84$eKdX$EF7Fp*|WJ zf3+%QFmu44g=J12I_RACkBEZx3(=;|12htRaIP}^#@(%p_B&?OY(EKw(Uqf9gJF^Z z73w2PjXAVw{qMUeauCP)|F3Qj-M}0Q(Lg{L=s`e;{*Qo$`v3TvkLuEr0A*vBkq2FP z5R(mpBoVEV1ekFm5*Zd4IUW4sxy*e+)FjGI7c;b8Q#E>Xb+xwDf3jKcG@!bS%#vn( zU0WyB%GSENa|^Tc@4ByVvE0u|Px=^{=kM2#*Wi!s&X-N+nR|hg9Pg9RpY|IpWcs~- zPyis~iHr)-6!yGqe2D{IM)^G2*hv#nvXkV&8Nq%oA6603Ok92X{3Oi1%^#Hvhq#m5 zO(Oi=#-j96N5{C5Tc3P`JVz-%nLroJRDdo)(U~rCQD?id7sj|wau@x)hhtnn9?U(8 zK28yjK1J?Cu@>>v_?=7C48koQ~qqN!m)991FVa`m$%5o zBF^efZ@vlQ%yYZ=&Mta~z7`YW%*R(Dj^jQU#{jKps{b!3E~s5{EC)c{YLlSo$_8k) ziF_!3oj`$7ls)h480Y3p)$4tToosFvXM%_5Kr*`JGse}zMNP8?DnIt1j9eyhuubF| zuGZOR&(}TeUF=yqd;_Z-m09HZ56M^XV1liOPcmgzzn?G(nh?cI0I!I15MFP-t53u7 z6FIupbRKtKY9mbiqGPg!|D4rM+ZCupn$DIeoA3H`Vco)M*Xf`q^3YNZ-@MVvZe!Qs zAT!khFAMo!9w!w?O^I|{Ysp3@-s8lo1lkpP53|)BYc2_U_pj`?WkKUYQ<^y&Mb&hL z{Dfs}h8lm$+?K7jTC^1sEBIA-jt84j6YG>4sf}hTL8@8@d}rk^l40$_eotV{Mi6~F zTN>NerEO$MLcGk>Q+%1$!m>FJ25d* zuOo0wC)JWOoq}C_-&}=e*-Co@4lV);#a7%U)JW`*ck8UVljL?&b|o3#K;+#p(+sDj zogtwqUA^6^Zo#uZA3uIN$qcv}FY4o3e()$+AZy9uLBnsRK3j%Ap%cHH~-qQ?97TiuYr8k^#M4O?)T=H2~*27?-cu+aJ#nq7sSd*&ev!A4^su~-= zAW>H@Gs<>teyx#g^%+R%4PT>QNWd|GZ#Ar=M-TRNhRD%jNsuYmc?2>I9{n<`WM7&i zuAAJ@n>ajU@tz~}x9439@MMMu-Mqc1!i|E+sO7+cP^ck{RY>3|eA9I|q6=PgY&{Jhln6=^!35 zLS~~c$(g&)nM}on+A0~h=3QddeiemtqD4xJe>S$v!yQHGr z0c%B`AQu8+uiJ^uPQInh@C=%tB3ph1sk~1+yA0_{huZtcq*W*97UUY|3x`2|=tVs5 zF1^MvQN?)9;x(|E}kGLG)aCt}it6A!cGkb>9 zg8-YWzC@E)vvNk$p%V!g2o z?lfG#ae#U!B0izhJG=JM&ZX0#Z{pcE1(@j!JPKpD)AZ>5un~!{{a4Ve*fErsOf>2n zVZgxw@h(JDjFy$4qQ>C8_eXu)7Sufi)BC!$NL%i8O#Q6l3!N|;6V0}%YRp2Nlg>AS z>{y3?)8IZ(3y_p(B;9T(GGzVrt(5-L5^ws^Hw&I||Aw0G4sjwc;AGOwT@gDc!A?Hj z(&TeXH(Y)F2p8Wrn5f;a;uHS9e}-mVFR!)%=|Mxn4QcWrRC(qWMKstG3}Bnl zqQ%nQzNsT$qT+zE-gw+X_LG@ThkTW~K4_*xG|lrc-03HN5AxF#Q%_6Dz8jgj`2`Ym zN2nq?oW%~#1j{=aS+KTlvJz22T7Fkvi8#K6|wI?w$rBNqoouB-P)hHOQT;kUf2AZSpxcp>o8Qos=Av z>}?X){}>G71DDPS9~H8Hh7G`A6tX_kF|v&DgBrM5>H4A)9LLTGhgtt<)Qo#20)}<) z{G1&VbmWg6)|bu5y|NB=fP?W%a8{4Zcg+1t+|3LNXsybs*&s!i3;@fx(#*b1J26Et z_l#%BE6UWQm>!IDjtHug0wd}0izVS#Q#Xr_TbCOD?d5FgKC1d9PpZo8TSyrsm$i#z z+HJRdj(Mxnsw{8FGtPO+CY+VVlzvDz4L@Wk&dv&ie&{xF-AAq?AQ&1RB4qNfI}XdX z)uZ>kN3s=O{k$r@+vd&|Rl6wVt*a3FW=tNu(YN)2a5AsYtFy4%l%#Q25F$_!_MH1HL7IJGIS3hA}TU%nVgIu1sz>R|QwSo82KOZ+yH+B-w?@7KeQG zH$pRMS>ghgdFO~Wa8>QoW~xWi>e*R$i0A1cn-hBt_QGp-j2UC%eU1Lo8WkFs*wDpZ zn0VO+H-(-cjqFIjFg~J?s*VHaH2un#mKBd&>u*ZYOj6;ff*$5SPv~>xF-LNSmZV6y zlkfg9dG_itQ8u$ol$*km!{Nxa3)4?WCqVOdd&D>t#7G%3P^xBPv8Aqb4nZ{A_1;^A zjv2du5&KRQuNklyZ*CPn5gZ;7QlzOpIOV!{dlk2AB5JdUzY3>nb8LEFkOjehkKvl} zaTHYLK76bAo+9G~j?x@`PGda5cHoCd9+IXxwu;%Uh|=V!!lr`yYP_*rBCrDbZS9h}@O*EA0q9K4;M zCCak=muNECDZJvA;m3gK>`nmPm02Wz^PA%sRKpITEny0LxOV-CU}>jr_IqI)<<$7i zq`JxXGVDMdB-n!k^V;lhysKOrusK796PP)|jRRevQ)oy$`|*7xjm8S{iZ^9IT`kl` zq?EHIV=_v7tnedfM^qZahz7ymIGDp}&>Ki6(EJ>Y`9Dux5jDdPaI^X%gkm4eK6(fU z`g|wJ@Uv|VLNG08Q8T3yza#~OQOCz)j{pbgd6o|3=O;ru?7ty)A?B=kCl+VU zc^NXSXBb^?XX=yB#>cj4fd_4ch&$0eAmY!Mvb@sya)E5!#Xf%KY@X^K50ne7U`sv1 z_S@BLjVy;3rRlU?!P`0^-`dFco-p}`Re>V^DlSXcprmS_CuBx{1jPihwXX1s940`3 zQ{~T|>aVb0#5eUD0Hr_h!Ck@M#e`ha&Xvt^%L*2r_Jw!EJBMOlZ1xtTlzKzFwYLv_ z`IM&ZZ|=gQQZ5|Xu%I)=I8TpMjaPZXP56qHsA$N7C&q0ah}lv~+JekjEHR!K)%j%R zTB@-pzoeB=pJwIHi`CBy?LeAhw_yc=J%n`^!QqS=6gPfSupo(yvzSI%NFKi7J#Z>+ z2VX>R)pHEm@&;^yMl*P+}-`5bL+EA7s^*? z&D3}fh!)@IXDijBIX#@MDV!~lM~W%^+PU~%AQj;bPVgJo)Joa4Tn1Br@)sL0F60== z<6ZQd#bb6w)svClkVjAjM!M@o1rS+U@J5x4UwqHYtn5GOOdCQYf#`W%9P9}Z29pu1 zy!0Z}WQi6JN7th=M%0t0a4_o&;bj0F{X)E3^9!jqe+i2p5n~I^TQ>-)L;b9hY;wIy zq%Ub{Lur)*QJ0~>`|Qu^C7&kH7pYMH9b?$lObfe{(8Rb?>X2LBC^|+?K>oq78`<0w zgon%gjW`8OVoB|fEV~&xcvI234mEMdBZW3x6T^m9ovsQ@3TLawTzx_NkjC03*kVNg zQjd`K8p=KV>>488^#078r$9<){4nWhN$)Iihg7~fb&dr%Yjf{tGmRlI1?Ea6*V_Tzbub(gZyT~fXG{4~XK%nZP zvi@xhXvNx~T%hX4{=eLFZ9d40-KPMrSG*d79X`PmZciWZdoCxS;sB_&fF!QYud`2# zH#@vMIq*f*=&l~@mTfTH_m8t|`X&F+a;hS98&_j;G|TJKC_u@88k^u{@)*4vDptmEGhyBh<|cRep>W~&8SQrzctsu?RCWkdOs=q&XgcdO>V~$cb~cZ zJ|Ui?35^UqV0yop&A&xGn{XYr!naPLqoi)p1HY-Omc<$cU9p#SrFG5#ir)3xZ;vYP z*gnqimbv-4h4un`A8`DYz2Ax$;E3gQ;zF=+U&8*~NM&9?2H0mW7>JNMJE8-V&n19r z<=g{~?P8qbDc6*EE=XUdjOc&Qnv9n+sP7rli&;<9WlzS~%FEv|N&BXt#n;N7j*;o3 zob)F0@PXhSZ+zIzALRu*?Atl*KP>#kZ5CCI5ZZ*pBUhBFIvFtvD(sa*fpR!@Lwlwc z+mDoRqn-k+WYP(fIH8zX*Fs-KI7y-wRG-ua6Ge#AXj#r2J>hls)}}~ z1j4{jM<47n%I54nmop6$DF0y=%LJ9ywTaF@1ug=~hlB=&?uQx%F5~ej0#d2K-IyuO z_o2!plPcq4Ar>!6?SlFj0$rf@N3F1#IRwW*8Sqni#EnDg2^O8L_@_KU|KVYT6z!fBN%{F*s1k}erWLXodA{uq6&wHNqIV<%F*xt04EV0{= z3qQyhogke{xu@(NJ}g#I0w(lQJ}VZ{fBC=IyBG8ju^}E;_9pUSNr-+B%DcwVe*u0< z#&Ei0xOnB&aE3zS)SL0LCVqFYdj%@e;dufRhmYXjOF2H*35z#8ZQk%B6nsK+HLnP7>{V6@K8K4=ykCCx;nQ;Wt z92(_QLhV)`0P-)GW$3DtlTGXq;WUE5^`m+eWHPU2zd%O`c?zy!w2GLr{R6RW&GjZb zH4PoC8<& zRy(ReJO{~fEWX?|_qSQJ<&KPJWEu@aOy*YP+NGw&*`r*>w+MxjWyYgA^g{F-N1984 z9sA9S5bhD2nZl5ieC_L{@9^*{)&e23@VK$zTCVt_M z9_A$7vjwXo9`U8u*Em>-4|BMboG*4pa&)he zZVT;>J4b-=e|7QIVO4b9+i(DBI5Zp%N_PkdA|;a24bsveT_SmC=^8{Dq`ON(x|>6H zNrRvuA;@`<=IOA~UVct>I*yiK;! z8_mbxdc}+^_b6!%u{KlLec2)%KAuuxuJ&9cb(cN;_M{j8@`Qtc3tQl+>DDKhnr@A) z`p}Ddi(SxGWO>Q9hHMr4G@;{VXXn+~@Y%x8jkC^$p09^z3mcQ-7QX}h`iC{YH^sBE zcyiahM)Eu@^($wQXt0%P&`yZY!P?C*O7e2`d{^XK+-$`f9N%PHv@Ml040#-Bk^6W; zj`dX#<1>SrF-)tNsB#Pp1qw7Z*g^7ytJ@6Aj)sfuz$cCRu`O+4_076Aipp6&6A>QM zq@DyzR<|wlCwHG2063`WbbNqt??m5p|uOOQ{2kY`PQNBr_sH z*X3FoDvuIxo!<>XICZJj#yyu$pt?055^5D^CX5utnwhrH_MJI33$ z_CwqIw#~vUV$@ImxPMV{*q+9HDM~ZM_llI%jseC|VtW}pXzyB}t zJN;0kog_R?NOZEc2M`WZ)`|LsNkR1CbuUF4$tfL$_05}J;DcN*>$bUYIQ#O^azBr6 zfVSC|YL?#0u*u|v9NKiksrt+ng|E^p*?~*63dR$8iZ+uy%@*3VDcBz_&)&520fx|#*VhPF?XBIlu!>*XO(p8iY{00kmmF0s>(4Oh~)ik=bReUwx&q2cv%wJ zR<*cj%Xw{YkG*SmoDSnz>7*(qYGav9cPqq?CVQsdy=s0FNW|w4Q=4}VP76~bk>+FD z-<@^cd~|wbC&2r;xp@0XL@&a(DDy+iMG{`0bSn1wTXmE)SDOm1nd-IFPxCO^v3%F! z;;IYR+gr6w3kymiQK-%S2#23mOS9zB@a8U;Fy2M2a9cBcbs0u0XC3Z5pc2|-#iBiB zu1@5bdp_bIsj$+@xO4~q+EO0x)E4WJ?d^PrbhLQgx%9H-m@G8P`40O9C0Y+-f2qk$*a-af3|fwI}0gZ zsIbEqIAOFmCaSPZr-b1wiW4|rDOyxt$M&X?z-%pxm(?{U@w z1#pp{WNT7x)c!pwcZ>I(;+4gNKdw`Oj7{wOrl6$+oSMYI;U+yI{N0*=+4Lp)_;aTrg4PV$6C`{2#?)dGBqDrg8JNer zkLPkI=W`C^2714Y^W&A{p`wfPooVi`;Lu8%;@qele?fX89grQ{&Dg6y#_5@Mkg=RQ zr%xRtc)||5bx%?8T%(py6H4cRH|#%?vIswWl7oSYDbtH1OEa1KBk2itgcfp&a(CF< zKXzx@PK6;|HMmB?*QSqe<f$b9Bm?0t^mP*5>;~KY$-^_53 zQKokQ>tdKqc@6n|imLPT=r6j#neZlhsKbSa;6PF!aRJdheZlsMOP86jjw~K}MmgpnfyK*#f5igJ@Lmo-hKt(5jjmb0_X!PaP zC6O6oo8*)@?;UPmnAWCm!9h1%`e$^L zuO&3Zv1tiS$j?kLE(Ahe9Al3ZqDFH^qeC|dTapr}*J=9bqFElrT*+~uX5+w?WtaP< zoI}z^*y0onqv+4dICO>>H5GLdqXJC22R!wsHC*D2>u2Y09I)lnII1Ep%D5YeJhhCi z2mh+`5mV6dzS}rmy zES)F;XB>^rF9NN}rbAfT>W^WV54)Ly}c_LU+l@^augoduc{E$j!8y?(Gh!D@>Quld{7GcBSAY==N^qU${ZJ}fUvLsydHfj zZCU%e*4d$eXN;c-uL48)WgJ_yD9Cb#)vAX}i^iGhxM8)EyxlydV8D)7q()HIIdD7G z9@8BysLfxg`zh%3TR08g>0CI0+bK_k@xoCjXhgQ*4E>*I7Fe2q)J*wof-oZ^6p3yS zi_Hya_mhS>FKU4QMwe|hR!<7%%TiLWJt~m(4s)?dowJsYHw>dBTY8k{43JD0R^>&r z!@q%I+Vv7Bl8AZ6{|2s8eIO?P{o3ASR%z>L`uf<{!}*#&M+G7!x4$xtd{Lf${K%tZ zdWwoG&l6JF9|zDF>qwJ+?rIDufj$Wz!RjkkX=apw^}0n<(kyK?GHmjT#&i3*MYlvU z^9x`>-6^iq9U?uwLyXxS=07R)Xf>30N|!*A`AW3ZM~+&I_uVy{%NJiv8uBZ2eeHB~ znKZg}!_{tC&iL1GDg*fxh^nSF6R&2Pmo^lLmgW&8LX zdTqvYFt>s^=I3@i%ngd_4YS4XTFZfjY_9$f+7iUpZ-i8i#FZ@BaZnOi7pHtun5e?7i9bkCxRUr$sYo7BY(OyOHkadI`Sn^d zzTpJVO3!N1!qctA;X`F`u{gX>b$)9PqZ6a99_AwBbLgudP1`j}Ily)Ywpd=m7EUMV zDka}0-)LPjjg8Rinqjt-6vuPlRlq(bN$Uv>Fiu-msgm2lk{N?nS#p5)alt?}o zdYA!2Ytm=Gs<6}R?q?EP)H>MTWtz4fz01U;bUsXxb6$ni`o0$HTE-7DU?nBH({tKI zi`{Yl9`^3w$PZ5_N$9Ulfy{PhxeQ};L3ibfy43!;>OgV+-npl@Q^@04Z~Pw~gTJO7 zBj1ID`!nO%fr{UeEE@}~V!?+JwB7Zj&@o_%r~#*jU|T0}<~slTwu9b5N@kHVv}SE3 zx(d24&Pqi(>S2j_oB~gvBU8S6kN2ImFaLC9*8XnugMGbgE0m||V>Ou)!c`sWp(Rjl zu_loen{s~IqS&lgi$7UPAODgj_I3Vyf*b$e$So3^vQE9*6Mdv$3$hc=fsnn(Gj2Ip z{d##}VtnK}N}7Z2EjHBWS&3Jy+yL>S)T;D`3~ENs%X8ijNL&Pia&(hFlFqxCzR9@a zJG00xItHA5vh*TnmQt2$+t6T1;uMuJBMapE4d%#`$<@|%5OF~vR9U6_{8k>7H$pYB zi(F4S+(0_oAg|tWI{4d~|H8*|6A*w3(<#ljA`wgSRm05f-)1hSiL!nt$7>(!-xjwS zD!0Wsb?uh=eb8T=UM6*Z*vGS9cNZFi zF02iuLchID?8K&4`qfB(y~Q3sg*S$w9tW1pYwTy;I*!N}WF$aB5&%K8{t&Npd&d)jvQ; z%SNiSR;rAYjT2`e7B83A&0;cu^0+xP{I#m1gcF&tTIrhS68>Gt&gJtD0{gAJK&~Cg z`fBU*#mbWV#W@4)=GS=xlJ046q~{Vt4zjy^ql`2Qe*%%@ra2v#lObWtZe)|JIE*I% z88akOYK}bzq<5JR{zxf+wyG7WO)#((#zgv6@mIZ>Ih+Uy4L0KkdMH zY|{T5YboHPhlgZmj`uCO(Mv;rrb>fOK;b$e2=OUQm-R-X+&HZe158eof0`arT&FWm{>D2jpQTe$ni3*3Vv(E+v44rt79F5SA`L zW&;a3@yk=yTq~{()XMZWriw7--8JCOn%axmB`HM~@eRqk>YN_4_DiLGvNfMiu}T`` zJ4)s_Dyx?@&O2z_^%51RCZKNp^&wF~87j|p>LD+KcP%h0Smpv%(F}GqD#>h!TcY_0 zSHO+Ag$>QPeP8r{{tAXE;OTa>1USxkJaIaz8kJN<`*WHf)Ii)4OOvs93r5I%`Uxnv?awurOw!XOHwQeQR6)qPV-7sfMRKq_`+EZZJ9Vn9i zZjK>R-*25P{Iwb8w;3^3++yrg94CuU(bpJbuEu;FaABnBBk>ozl994R!6g}76B;+N zzAxLIyu|ou(r^33D?l=3<^6jf318hxI8oSA&MTU#5<51L+R?RVsk~uc6{Dy%jL4zg zGw5aOws{;c4CZ+L?gz@X2|#xH6ex;6bWaq87o;`7vwppYrMRsqxqvm4T{$~QFyL~V z0JS!O8Pn}0OHM04YjiffCr;k241r)kwbda;=R_}EY__5dIBb;Z?Uiwi3}qQM4YCQ> zVE1E<@L6u<7Jo<;K3VAP8`_ob>7bgg(xmW(j|EFC=<0%{-U9plakC zKjw(vWS|ew@E=d@Fo-}O)xg}g*sd)N_dGwl-KV)$DJnhCt@N-YEs%C&& zx79Jt#+!W|Rf!e94`lHIVkvDgx=vub4GGuq7-8&UpFJIH&q}OWz`kS;Dn5XQz2}W0 zh+_Br!vc!@>x^e1H8R|p3^9Pq({Pqx8FG0_QwTjgX`(39VJJ`!5DSNYi+Q}D&d_et!n6oz;?I;m}pg91J*9GArl#z zSt)-Lsd|dt8t>cXbSmUa2wT&3>ingh;9VW@vp#i0KYVlR>*GO<(J>-_{KZNX8z=Qx zG;DBm`WDmd2$L%3dSaWZ3t_msKpUZkRl}oog9S+imU(I!(9ec%u2DtrF)MICr*l-@I3|e{6`Ui2UQ?MzN=S0c`CH_1dP{m^WLsZNTiPwj zVcV;Ln#G0=@c+l(TGTH9JRKm1K4AMqWZwVh$n~c~0pjRL5G1bO~1%V|LF*LGYA2>(cGICDX0-9R**L{LZt*w z21P+)YzUDZ&>vz2aq-=Q(ubZv9~449=#zr~9t7jW2X2PI(EnSgfXMm&?~&_YA^I;6 z1LyyiEdcBNq|gUx;{WyeXTsjU`qv8`kQ{yr($>0HqvUYz=j?xfT#%3i9{8c?+yg~N zZ~FWJD5V`|e*69Z(w;0QI>5 zCzcEV#uyYB8N~xJxdJLyYzUPV57{N9v0!wLOo4U;0OZf#i*0qlbTXjUOW zv;g|-Q&2@0K+{eO=pn?GA%w_@m{Tw?Gz$T(jRHHf7NFX3KzYs_^l=h6?4|kVS%DcI f+-rUk5ba|kRgp(S#K}lV{D^ZO(dlFOKidBT*sO)q delta 19839 zcmV)EK)}DA%mcu(1F$Or4XW8@*aHOs0O|<<04pTwK!b{dHxe$1wboX!v`W1o0WAS+MB5I@A&gFD(#gb2?-zUh2fp^DPhG2h z3AC=-)z|)u{);|o_nFB+5`wEN)|oT=?A!P4efH${GOj3?!c~8qhJQJV!76V>q7E&2j*mCWsE53!n}+H1!u7+?OJcbtmfIb8SHXLDUxwa+WwFgGID~=>&Ja0gScW^n5K1H$8Kg*^Ve#2& zX_-6o`m#xqXvWU#=A!Nx;=L}E+*PB(kj&UlF#LGeRg zu}hT8?q*|#PXF|(?ojr5+j98>chb}=m5i+yI0@svg~i?U!d#}|NEnwWYfr?mry;f{ z5~0QU40nH5?E*tzgM!0XOrCes{uycZHWT--9FP}lb$f1Tg7kM0SNXd$df8Kxu|mNv zKFIU3YuHvrMvqELhn@0`Fb}h9wO4zj*=B9|+M6&UEOpP}ed#bLP*`k>t&7PKW1?>_mayR?1 z;__1SMGQQ&T6njZyVrGxTQoD0!ORFEZDW5W21i3{%&$6JCkl4utB!CKyvLft`cjd6 zg}ak&#zkM^1>rhP+SljB@x<0)wFO`uT2P)h+t@5^u}QvY&_oRDo_&|P`fQ^w|6Vlt zs*93aMO4(hxG@YzTLwxSL>_8tTyZX@WFonxnPfsZtBdK}%=N|u?{1ZmO-Xm@ijaTD zo_0LmB%mv{LrN_`+mO}<=tktW&KK#EJV4)O@fQLUBaqf(^p>V4qi1+%4eVFi?7(qa zBc5yn2;J0lWIs%62sHbPTDogPBWca`j1S|L|=qx;t%jg8Sj*W z4KzjfVQ1#vbIv_?ZsynT?>_hmdy5+gJs-ykbrMv z#l{_m@n>Ni>gNmzKflG0EX$f;xL65efAPA#yCc*az7tWztH>&kwzvw-xgSjGM%bd< zhLU^TwYF}EScg@vrDAYj#<5W4h__mTFvW^g^`NeJEfPUT@n%z~;DzkOk>s_dvjQcC zsk+b`MDIvd8_0z+W?1y|mG}Gu4`QK%;h>U@y9^8d$ik~7)3vo%WSBb#$lz?sf3~WM z_0aU5K28;k4;N`nlEyin7$zH9Hw#VE@7tD8HtxA7AfQY9n>gk&z$A+{R$ZFz15@Oo zjYkZH|GP|v?1`~ciJ6g2Gh}+ih{yF{v)j^Qmtn%pMM*;HF2k~48GvXN#`RMEY>45> z5a2&jGpA!@Ld$Z4t2L!KnMnHif9*0uZb*skvYGJoh&C}#uf~P>60po5K`($#0j)Fx zjIA8N`brxM8Tya+f*)}SW@3IG5I2mk;8K>#(jJe$A{005jF001GAkwF%dUe*qOommNd zT*q1ef70&0r`6Np^|`*XPV89LX*-S`%ZU?9zGTaitd-=-cH%rqPtw|}UFGdt+e%79 zN)4qYgrhXg(WF4zKq=s~^~#OfCIsq0fpSxz#F$;HTyXezZt}D;kONZK8PTG zCy3w0?*;J;eqS|zpm_dJHGdSu4*ao!FBtffAeQ4#g9zczf_NTZRMTHl&7Yh2iy+>Q zzf{d%8ThjL{&f(~;ctTYTYN<|e^*6me{bR+g7`=LlYxIW@p=%O@h^UVsDJfMeQ4*CbG$tTTQvml+C7WG39ns zwwltQHrQrJqajTKt1FRk+|Ib2N;xS(sLxGao;i^ACY^*A8@0WpE2tanIo{KIs^{F$ zq5f!BZx7kJ&)XO6wz!>`Xp4GoEHSZ9P}7-Aq&z#}4cYOuV@k7spti5S_elStX!Km? zQEnoTu1e)=L3PLA;lqde&qcdVAF2czND9Q06B7>Qt?N#@6KxZ&Jr;M`F1hyfwBxpQ z>q&|+IPS5h9Qv2NA;(R{k_kcmw40o8om8qjmhzm0+NY)5J_nPR67i%x*0+G2I|uHL zC1T!wK}W+98Z0({eKBR*kigfO9HWwT-LZtzlb#xJ+yQ$e?kMLaNA38K?Z(tNNA!7< zG5UYQQYur$I2E|kp_3Y6LC+z8*HRf1Otl*Z0 z?7j)dYa8tE%1MbO+YZO#j+S89V`EA+rb{U+vt-Okd9g%)PF8K{S|-4u%cIV;n&jg8 zyv(kI=eP+wPUX^We8H~WTvnS-Iqrc8Czq)V{78CyTxCqfnGWicNKf@UO7|MtPH%bL zPGZ8FWGwSJ)|pHzAvW${u$H-I!qG0$*=i=ubK%X2>0s zO`mtzso3bkcy22juEj>Ezy(JOV}@KgwJR~6B&LkmDQEYtLy1vc0k=1l$*gh!Qa|B% z*+uRN$D2&jmurjoTxUE^X>Hj#@>`B(&hr}Cp<4=nPrW1OxkyD_(eCVZ57}-!rnpuX zaTO9N&$y?EF`y&M&g!BS8Zx`}1f#M`u#81LnvUC^Gg$D%t>pt!YPR-VLL-_v%}p;Q zU0M?=*-mGxU`0dO9fFEBJD=FCY99-i@u+GZv*03S!NYkAY1LfBB@5q=$LPLE&zo+YR$!qtH{?!BcH<+0 z)+OL+^Wt-da%7JocUiJm+AY~9cUy9g?6>ePyu-pz<7X_nSMDP~Qu`lJO@}3&a?rwu@L>xtVU8|PinnNgpIdTB4qI|Wj`Cbu!T?LUq#5>s&Drl&n;%#eOdqB0;@Ua0WiLjDQD`7I-t>{O&^VXIP>%T)aj zS~4W340($s!*be^Gjdh{OYWBeOCC^Ru!HP}`I<%A+DOj|>qn z8ObAago`3av;!k!Jc!)bNLulFFeO7>kfLL;Q#w8#+17|-TZ|wFm+)p=BD(u^E3;|OKN|A6gcPac*`0V zUo^uFQ1VwHyUfelpyHVxa#HdqpVLG6>RjyN;rtjjdL+$b> z5Z@_YIz znoN1wULQd)*RxfqO!iKu9fiZHs1CdK#FW0sO~0vJSxo8r-j*qU8v^vH9ZxL?RqlGM zs;T8o-P3bNt-7~*g~LwSsZm9_blbvP^1f`wm%vVVF_<=IF;*;$rdwL%+9-AJ3F=ZMnyYa#+WVr+#u-Rn9{74sBdI zM+(TFeWo{bE)^?(m3{NilE8Sg`_PO7*oh9#bh15&E*wT5j?m#pF~rdrjRF_ zq8}e6=f^RCS8GMzKjR(6`Z4g7N_xb(%!&X5j-G%oRccpVqrw5z>iX! zTD*dH<3||Oop=_HGjR<{zQVaDm@W^p)_;tDRh0TR{5X3-%6tSrfuBS*b-axCuvbHC zUc*n(R-a0Yd`hvGODXoUDODlWcoOeJrKq&duJDVAr)ZO3C-!@x6t z2A(zWegn@Lc-}z2ffEoP<=kYAF2yC9>l^5}NlgQb83|E0X-&xt6kQB_;3f;Me$h<+ z9~s!(q&;Q#Eh-#S{kV~<(&O}^Dz8m**fHFg!A@aw2mf~Q?@s>h=HH%K+;z23w*kH2 zLJ#T%$*j8+tk*i0tA@3T-TaS9D^=5e~?Uo(TVAG$CX3)CgW69^ALTdeqeglLwAo~euVIq6(yYD2 z%abgteiqauOX*P-(_<_o<*&1U^uQW&`~u6jlH9j3Y9FN=_LBNb_+>_Ll0MGT9%Iz6 z;zjoQ2@)S;Phs}s1z$g|{mLr{<$oNXppMGJO{lm@@s&C^Sqj%wN=I+# zgagrGvne`UA82M{v_!96V{E<(vsLgs_51+TuazN|beOtFSbbYreag0@S%q@81qjHW z(vh(kh)-+VLIi-%XxqYs`j_<$A;Kzpg*`v_*^OUeFF?*$wd7yLguX^qU|j!SO%v+> zMNT64ZL@_Sq7@2oK;iXc2LJ#G5R)-BDSub_e;j2Ue%|ac)6ImYfd-eh5T($~mSlU- z)}{w7Nh^^}T9PKAp(vBx>1LYA%sM;U0}nj#RunG?rzb^4DcEdNs(_-XhziQD{vCck z0_yY5>~1!jZEXEv-}8Gs@B4ke-*@)4f4}e|fK7O785=`3M`e?f&7^Eh*&K^uGk>NO zSTU%WR$#{v!<3vja+Fu`5!t(Pr63zmHbvPSk0FB-F`UFH75B=OkII#gsra~5`9uu& z;gfRZQ_c7^J|hM0m($NS<1jwgjB$KkHeXQjMY;T?7`}|J#Bir{mcdtL^MHb{srb5z z2UUDS#W!Q<#JA+ex23i3#CU**6n{LdU`D|s08+&;EMDy{kWbo zos^vK5NMV%S+n5vnXbTZ!6g|_iM_j9_WE);;WT>A?E2LP)v5%U$qN__efzGt! z=2AIV&ss+6gsbQChMO7-`rcYm>c{Kd3{UEtwrm|PP7AaJ&Me)|rG_bB=YOaW^(M{2 z+6@A$8+qxs3!ZLSQf{Ydo8E4L`x8qEF1&AMnYINZ02m;E4p;IcdR!_ENpP zSm~|-p4hNcbTdY9S6Vq7-BOI<-e+elr$7=67~Z6lRq&*S@8WwJcH(-)aWer!u5Ah=nPvJDf+wDwgcv{Z);Kv$% zf}d)5Mm9f_Yd^=c3V+UMcn;4CM7s03>uLCf+&+t0daVSS#yh0Nl7e#@=5Sua3%H=* zml}SB7d5lCeQhwXSBMf+Ye-$CYdcn&+!Euan=dVj&Odua6yd7?M*Hw}N6 z{%@0aw0fy5q3!yR3#?f(=9Ng4D*>zELXI+r=NI}tgLS}hD<|{))ST>^i-RMTGOnR} zeqIS|Z&j1gStFRKu(48L4De&PmTHFDs9_L@vcOJDz<2;%sncq zo)atyT%TxEMSttdVY6B2tB}Ko%bF533jxmM#JP8(;8;b^IH-G*ycj)`F$%2v8(8_% zmtD~t9Ao~jRy8m-U+ffF=tf+V)i<&5LFlZ13!_=ddt)B$Mv1m@7%ONSzLjYwm-DZ6 zK^V&QX{j*8FKUc;Y&ne1%0_`5ork~bH7e7s^Sxw#cMD2bhr75FK>V-k$B(pPY`&|XV%@RP@ zOz|DxZw#o+ZM2{fM_NKz}`)Jd36hmR&&X@HsRGGp&S{wkz0_ zu>2f9s<;{|VZ{vAtS_N$2JKuBaxvJrat>FW2{hXtff7EAaA+6j;W?}vTs?!SCH=Hl z{q%(6;S#PMlh)_(p0a3LoB~}XTtlG}Rt1}@rTKXHJl2E|4+qw+9jm~a!*xCWE}!q7 zNPj$X9`6;H!7e#^pTNsdd!lttuBVfDlxGRhlpV#Rb67ie`ads~Ek{bYp~U#mAAj6j zSKep}+$K)ro}NgZ=_E}C2&M71^}#e$p5C;;VU1dsL_~+(Re^YrZ zs;$~htG3pvpSJxL{fegl^WMy4k_-a!Blo>`mvhhZKg+%I+!qHA5z!p}$W7aMxHKcA z87a*uX+~#%qsftGjC_uDQz7RnJkCb^>SJzlbDoTim&W7f2|Q7nNp7CZQ`~d|PnE{2 z@JVhO%hP23$qG+*alV@#;28?fbkhVbaMK1^9i!0>0SlC^EB4ekzDUX-B_%wMg%jQa6?&d14 zcH^x^;T3LLh`lg&x-=`LsTB%m2!%6UTqiyC3O6Xc%EhZ)e3o>qanmwxlxD4)UgLEN zuUB}3yq@i*T5fXFNMQwcF5V;`=Snk2;mvMp3o^n&KJmnZ-~4Xy6F?XNIox;w~NIz7b*NrCbc#k)}vKH zEf&*bOrGkR6_xAi)^4t@ZCtyicKN!swW}I`Hm|N+yOJrV?mTUqRvy&Ct>ukIG!SlG z%rv|z5{?;K*jTRx*E89xB7U7|WL+SvH^f8DdUUOZ zL9sx@rv=w*(SUp>I_*YV0G6ASac8kjFbMA5zNoGldUYUXFfGa`!3OIIgSG@(<5A5B zM8b;;Eu#k_<)RZYg)e=asqnZ-K_WkYwvPsyO8e|$_kq_%e`MNc=n39`5rLj$$ zGk-y2Jj66QD56)V4J!OCbk_~;W}0_QEl(e^3Og&Zb9Eq^Vya(e)!h7?K)ZZHm%xeM zF3VyH?|@k_=!*xT-ZX}%6%3?On8|x=ZF(mY2k=)5OSYKgvqEr*$=39k?u$o%14dVQ zJ+KHMRtH-3m?0}$#OS%HJ!-@4aRYR9Erd~q8l27XmKK3}*2d-Vw&pHaUo$kOY;0|qV`?ki!Zu1LnZ^vAAS3z!gs)1u-Qtv$%@ws_Y#EKWL$&Es+*Sx!83}>TaOD5n?*4$&$Iz}Mrr!`M#m7WN#bNUz0m&Iot z$Kn$WqFJ4D`*&G?AiFF+VRNUuO_J2Y6P8vMH=42Ag1(xVS0>X`dYYb5=^c7krCxei zrQg#ZRC=7AQ0Wr-mP!}XH&uF&9#ZLYz6u+kP^l@4zNgZ+=`xje5VG#~RsI2At@1T| zt-{yI$Mq`zkZ(}=M|=a)@zI5vK3j^wBg& z1qH~;xA3hh-^RDAdKq`Ck61H20~zo3B;*XY>YgLI27% z@vspH>8Y5_wB>YD4sUur;GLNto9XpO^q4msF}x^04J{D%YT+(Siz1;$B$}0ZYZBSj zYec*)2;^RWy%UKz*yWv_n%7l^QlfwVRn6z2Tjihg{i3G_RNlk)Fl{<26N$ZJ*dpQ$ zeKihL-pdcFbSvGa@n{_?xHMCH>q-}3Uz-TMW51R#fG~_kfGy{!)?wy&j+@9%ek4CW2=<-6-U9y)2 zu+jv;$`a!c+bcz@HxPqzq9P*<{3+K;Q`4{jIP@wZ>F_j(;VD4oma= z0HIRlnaVHlwaD+HLV^E_$!P=2ER|o9X;Z#`ywXzmWtD%;uc-X01iQSUks+aiqN+$d z=r^4hwJ4k;S&Vwy`>RoJOC(z1m8kI>g@3E^Yy1Eb@#>(i#RN`XIqZt-!M1R$K#K{r z4lQhm)5S4IV3u%Lk2{)5VYukqDbv_Y_2K}2*S1A}BOTTm5cRth%>}i!@|<~`HxytIN85q=7*$X> z_=;luph;rakNMtXS%PWviCoEirTdMXL2R3+nUr|_MsZ_a>Z)VMS1!K>YVEj% z%Ul;awZ!?VGUG|fL<^EL;E7|GQC;;8#{W5xB3^lJHhZ&KT{WmeW1+^Km2I*e;F=TMUssWhtkBem>%SY!H>?c5*_u4(68QRyM~X!MLG|D-2AyU8pkP zoi^mI^c#iM;HLjvJSIao)X^?qLAi?3I|HUcEd%4rjI!Bsk0V$#FH#DJ#JFLBSaq`a z0}GlTwmbRQS7g{?6lAK>!jUk_!k{J8xPlB93TCJSoTH{D(-ql&m7;WiXaNKHA7R-< ze_901OA7%5ZxP35KDo<&WBP{@S3=V362GW3?|qh>5N3wl9!U_YPhO~{nG!MHbiLsT zS5JNq47^tFV!6{v7A)rR@3>qdc`xNT>hWIgB_gd>AX%L#l$mB67yZRaaje8BaawN4 z)-|SnUr8HSYzB$CNC%>SB378L5t{Gx(-y_G>@)_er;G=L_egR zkZdC4ype9gtZ6ifevCIK-Hg?Cth@zlaQwC8;S12`#>l0AIpg<}r@ogaG!^&I#0J{} z`{+^hu&ct6YtOosCY5>|-85-|J=cCq-zOy=hb&yseqJIn|jDwq1YC< za$O$hp*v{SXzKGgb6s;U+)pP}WP7r^X`1~u8sDLY0uyR9m6~ZL`JgC2okFcpM}SVJ^Jooi#f%`nLUcYw zs0)1;QPfmn3j~zaw?j$UbOz0*JLo6m5}{LSy_D{RlHeLxbr;5k5FNgt)y{3 z!6Awt#b^n_$*qau(!s;F15}np3C!8kFxP>$6JmA&=U)fnE}$xSXuoeXq?FTOVu{VS zeNbY57FMpLZt8(@_M=xd6(>Ch&?9QdrmQ10U7>?h28h^84<|%?2|5%eYD%A`s-lt} zDzC7Yir>t-k>&zYvp3|-QA|mS8=LItnA_OoC~a(Vdh8-ug<~(x6GYCp@23TOQm`p9 zv3vkOx@ZY(3o&w)l2EVC)hM!z%dXp1z~9sKj1{J{hGU~_^dAQ7PDKpb(@S|x#W_oR=(Kun=%r;%&PS-S$(FMm z2Ff(WI7D^xv<+BdY)c@u`B3IdeQW%D=_zE`ZfBlhgn~y zS4=n`P66OBFev~SgPnh4!Z{az{QNcr=NfXk`mnDnX?gswRA`w(uPL-rp?abtGzEQq zl9$sb5iM7!@eGC54KD=Q*XfN!25-Zcc+G^IE&EB^OU>Qnt1Hg&caxrVCpql9ZM#z* zoMW>4Bv^ln#sOmE0WeXbp)h?T}8F z`Q~vwx(7mTLYj?&yC@mv(+!|Y1$P;$x64urYyj8@mT^Nhqo4|Z50o*T-iGqtp;IC0 zGI|e`-UqD@kh(tvr4NvmK14?P2=qQiEdK=5K8E5xLG|+wQ`u{vm+5pi{e}Jtjcr0< z@E-jQ79WMY_CEa`J40tFWnTk|RtCEUbOrQREP#-Mk7_<^wBr`=L*!U;?E0HN~ zMxVenf3zKCS3_|r%B`ja_M2!#NvT2*B(@sM^+_@vL0_-)R32@%~d(d;dno zi{wk6r$7m!DNW>K?mea^^67t|G0Ejq&7#Hz$mY@inuX4P{ics0ha>*)JwmzM&-5r4 zcKS5IbPZOCrj?>%0Z>Z=1d}e=8nbm) zoB|DRZW$4_0RRAC0{{RxlL1>Jlf&5#65H4Rn%86N+c0z#vKz0e2(%4H+bR(O+m%yxmJE*!Xg zuSDA;P;?6?+8Ln`?T+AHbKb$Cond+uPwFx16w63Z=1erkVu=r|XE?}uP=?j9p3z}g zSMg-R8uM+siqQ=USAS_do78r6Fm9NPCOmx*?A`}oJ^*&`%-ZLy8?2DH@|xd7e*jQR z0|W{H00;;G002P%9ZJ~Z76$+TTa*1-FMnxc8&?%QV@n!Y9>j zK->nrHBoEX!CP_C)*V|Dc@lY~jz){g;JbqVVi?~G z>MHMe8I67Te)AN&N$+6AVvVUV1ECpKHvJ877ua`G1`%v{;lMR#=YZBYl_5^#yx+FmOP8k%Vcs35$mmy8)*0vXQItS7eG zg#z8My_G6|aOF!^%%XzX_Xswb*>rVWfkGd<_E??TTr-M(d_pA`f(Y}DzIG#{ z7yUwMw~dX$O=D`)F|3js_JHL^OJQ`BG-K#jf)~Sn-xJGlQj2rxLw`ab;a#-zwW8kD zrtu=nK*bLfL}+)MNplm@e|UiE7a|}z=E3PsrN+%% z+H#DXQ2LW7jAC2pyr$q-gUl=FsqfB}Eyvt2_Wap9Q7GrLF{|Lj)>)`A(<+vom{%$z zV*Af@m%?B0x`sQbXsC+2cLXl>cJ3L9j=D3mtJjS+XW!PmU*h+_!~4s_@xg-ydh9^s8+U82 zGv2V!4=RiQBri}_R>AN)y7qpFNfUDlyFQ#g(32Y&TJr7?+nC4EUw9nG8g}dboxc&n z-Zwzo8yW>C|9?*_^4dh&ee%}G<;>xpAKJ^p(h$6M7Kjf@LkpFQS>2(zVEpR1FbZXT zv`{?l?R3S{4KtHy)Yt1f3+r~_mNY=u(N;e4B%d`lXRnFL2Hfd?OR2oN+eMGZ(~WVZ z=lfM)JYClFr33c7vK<|~vcGb-N{+GN1@W?7V5*$0Lw|@Y;S|4;&hm?_8QpjQ=b+$& zTs2{k>ksW&C;4L&q#WiRdm_h&xOzWlg>x`bh4PxKdVynvGth?s?!`waX`T{3iZRY& zVB9zGFf~OtA_fx4J7}s~IYL_CcU6EiThQ-XI__!vmP8U-LSh1wzRmvB6L2m19e&c3RlsxgUE6ftz2mpv_+3_=ninGpuLKY^TA!+qx*ED z`*dAtsD3E4gJa8y?qGMQiq7qtJ3R<-jMJ4tv?GBNjOql2u!&*UwM!o9nrpW)#qh9J z@C+_87fn2MlUnB(mJ_h}`kZ5ECK$>`7=j?}w144IN+D|UW%7ej7#9iEz zAz=5+5Yc*eaB?|7M!i+^bqrK+?{;PfvE}F~W~%>9*YYW*{5`Z+kKGu@akSFs4ko*w z$bYAaE&M(}|3m1-WqgeIt&Gnc=6UusCEpW1?z228HhUPS`!Yk4nty={eFPuc9_kA`Rl)sr^yBmx^f5^PGRiO(Ve!KW|UZ5EX++;F3SV7_? zgcGl$X*(HBx~mSk8_}V=7_MSuTe9buRW!wNa%<~-yO-k3n+J$LPS7Vv;z_1zg(5|y z@44NRHR6-V))GZ7N4ogG;TGvNrd-OD?jAshFa^jR*WQn8QQaknNQ?Rd8MExl!BHR zGuGE-eI={u>YQvARda~6hc#WZMi?eK-PQlqu-PkRdP7}{48AdAMP+CdjM>fgWz~Ex zFBgat&KinbSd>jooJ&lfaF^j?=9Q(IJ5Ftl1(eOVQGfWF~-p3(2gk*T7>noGrKCkkrM@3&nFfvQ`7T9 zm7Fej&nvLW2d`}AMdB*(=*tquTCT)9>DxOxB0#dDtj)+9_NU-@!mxHORp1_L z%B~d8+oV`hVo5VZs=3D?Ef|}oqK<2#d|E1WdPJ^&f(YY65Fs>iJPl$T6C9I4OyLqo4;wr8Q$7ZOIT@6eh1K)C-pu5Tfa|(!BtNbY3niWfq zq3=vRR!vNf#mts$s)u7bPLMu*(@^cZSJ8<@;3!d@$%>*Di;;mORcUV(jnU<_>87O> z64X_HqQed`sGakw#n4ecF>=eRs$%WkEoX7e(cC%|+PkA>%{jDV`OvAhd8ezId)1=- zI>X>Oxx!CUFO9(emAd$8P|-%e{6=~luuw=0G@`uli1`NkZPICy&R*Js(by+=qOtf6 zydUW7K{KrlLdGUg!zK}6Q)nSx3&}_ymv9Atuqm2M#ACQl)*RIL@WV%YCi)Sc&+x^+ zpvHsmsGaTpf$%Q)Qj8@2DBD5Z_AaC)p|^{`QY=oaVE3M|e%p3fxDpnCYz9qi7R@Y& z2%ERvi8_Y%-O1n<+Q|(;-6qOIK_?Nx9m44#t{?0-A{@0Lf!SGdI7d=}!|e>PgACM5 z3Me*SW{U5d;^-o9W>Zl+2ZD+rZs)k$#^}Li+DFN313kuCiF1SYn?4D51w_bCH&W<_ zc!i}fMDhUb4&6MBn6sVad28Ev?jgDLU9#rIU~zWUZIcBw@7E&At}?O|2osR z=-<9WJ3T8oU}A$zCNuq`-97v1oNqs!Jvx8>`|Aq;b9f+Q2#Y7^k&zL>B1cY!ge4hS zTn^$2u5x@N7Rwwf0``Bg3>nurt_N^~W6owHmsWBlMDC8uk^28sva*DPdS|*2=ndS1nh`63*8(wYs5NhFG_ZlAy~FS-hKCdy!mt1UbD{r_BY@B zIQz_;)lqd(K-?}?XXExQE^~<&eEyN?(QE^LRT00juzF)_Vviaw=W-%Ek!0gRYoHho z^)X}=J=@ZnziD??fWfUtl;|gpium$Y+fO?uC;TifvKWes!Y;R;g)Rx?cEsIr%>B?f zkgSnjOlJi1sIIE_p?FNjUE+d1m(FUpy1Wc9AiH|!{Or4y@fz#7RrbvU-JzqKXuwxe*`Tb zDB`x1%yz(k1|C*wb2+4zJ`5EB zZTkMuy>U9P-P6BBmcOSSDt9|XQ5ImcKp#Yxlv(@b%!O#0o;+{gLL+%oU6T>Ky}^#p zUg-vL!b&#vPFhbBKD#N5x42`1^0?D{O$M_~YFx;x&7|j+!9|+y7rx zqMf_a-rv;h5k<>5f~gQRT0ihfmZ-MHQpc)%rT06kc_Gq#3vj@vOh)V;xwyG1qGcYh zrJAH&A1>Au{`-~cSyFIRpUht|N`>xcRAXR1t&E=v?8AnIt9A zO-Jaua72Z&#CC-3SP0!P%3nMK`JA_xy9z(mb4oVEa^t`glO3xyA%Z44Yz*e3@O4i$ z*NjmAAk5L@xet}A;P-aThg)>Gk>GyNq9G(*?gXY!`mL#J%A?{ygGL?>G4f~YID7R3 zT2#XDdg(&f1y1F#W3P7i)vhGB0X&?a6uH@%t!oDYTrD{Q&VX3bl+C6;KSg+V(gmFG z)acrvU%N(V~+m>AfKh0UUxo5ki%Ywt*l?v`QO zdcPy>oklaHnFP+Nd7w$z6wu1+QY{BnvN(-sZsU$G zt?PRb6r7}vufVLuc@vwg21YB$6I=e!)F2GI_ltw&eR?B|YB@z%Lb-%?LW+%vr{OJS z?(6UdSAvctJC%o~k&tgHYz@k`UOl}Ku2EM11gkY6^42SMsDI;URDp(U0{hsY=JeMk zuq=i6o3t|3L(dhTdb-%SGpGg446611WR}sMaWDE6N+`NBU*YSg9`#zcQ)0u;y{|3} zWJ+aYrmJk5DBH%JRtB##v$5xn<^9%HY@bG!Ogm;e$GB2*iQYKRQ(=F6^LnXFuGDw8 zQNl(9AM)*@JRi5>E)xn&dqHPUhdX|~@=$|5<4uz7Bh=-Vs4geV0>bwaMqelwsMD-8 zgF$+4G1t@p;-itJvqD;_t)Ep{Of+<*F!>SJ{Med|-NQ z2$D_8lH62!kqG@)r(x{X$-AbC*>B|w*%q&2@P}MHyM+}u-~z!(zPzUK{N1_hCuMxj*9vs`@f&KD$d(m6=4-h}by9unzpV49=Vi?dU0C%xp%;M_ z7@zc#p{~aH%5)VcmewmeWfcZpPB@yBm5wAe#e7la?sI8Qm{+_lAu-SXr=pnzZq6TQ zK=}ne^rod++kY5N+pf1rL%mVm&3|k#xZ+fjcwX(;oR-kad}5-`Y)K+)wIEUFrWAR@ z(%A4x(CfJRv=f8x1m(8mcBJOtgls0=@V-CR(wX9uFT8VNM`Dg|U(q?CSDD?Z?ZPwF zBodFG{IjPYj~{yz>0xIT2`Ac~y#=kfWOJ!7J+HFH-muoH?>W%~_0&BgrqgS9MSLXJ z?O2fwtbC;WBqK{BSn@kGp3@@V(K7A2Q}ycOYf`!57cM9a#=fXR63W-s@ypLK*@x1U z&B&m!ubh$vam$3<6217F?8cZne`N*8C8cQIM}29(sUrB^v&nnA_u!XaE$C>P(C0wB z`mUwA5CNuj^P+~i$gBDF&htlIpE-Cmzbot;eRw60KYritlI_HJ>gGnpU^i~+^0gkh z64#rJK?i*`8jQo0A5*t}iKWGC{#%kG`1MBsezQHOr1@0=14 zz~k+PB=U>=zUfhg4Nf?Qr{DN472oNE>6uH|&udobf-IU#H#V9ZHD>Qpi7-x$_Sk<& zW5wia@vQ?=r?9$4&5<}C{yI8(G@j&dp>V&@Hn+V+zZM}aC#p3LG{R|i3k1)=YH0P+ zK9tkqnV00s#y1UXQf`hL!~hw4p}SHM=>gI9$mZwoRN2;h&BBc!}KGv>Aw#bMNrgb z=*}0azjA*QxbE*3O)v_2xlYyiJfN@ZK$|%0X(D7EUvuY9t60Yy&LoGDg-gFS1%55D zW3FX=QLi~%B{>j4+E@x#!+f{w>z@^&W(P6_4C@HLysvnW>Nk? z+(>AD(x^h6)QBc4&1{YFo!8MzdG&58x!5DFMpvI2@F8axj_%i}1HBLWa3Afy+TUj` zTlR3yVoklb8}-9#m+NqMhT7qs=ri=0bwWrqDgNlww$25xK^E$7k(=N{x{LR3dsuoQ zMD+HY^%ihgNx+uqpqf22s+FU5AUjj=fXl_45kVvb;wcM( z9A@^blC@y<7lc4=)A;WIzi+qcibF}RB=yasx4Hk$gfdivB zpiBr%5my1_#&AHj7*iwwdX4D=wNO@%-e4Y7EG96|Jf^N zx_}(sBj7h@3M4>T>T$sPkN*@xH3Y&gvEUF@9|*Mn8*+iiX==cTE9g!$2b76m6U`b3 zC4$IFO`w7ZnorsRCq0;h5a{491ujnFfKS&!pK2_qG^GoK`LN*Rq$pTDg#o5-vcio4 zUXV5c2RWuOfK(tTKCOY>vpun=P*@OCAPHtq>jU+nEXcTv1Y^hG*nj#gC7{ z0hd2n@FMdp12Z#r&+ERY!2BZSGm`-q=W)P#131tn1L`c`0FHLhW5EWH?E<%k@ZjbM t97HfMz;-t{JgmtI1DN4(FPO?W14#6Na}0%pN)RPT7Xkut?fZBBe*sCPA