From 5423cf0f11737db6fa19f38c0825a8d8cc9a9df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Mon, 25 May 2020 18:46:10 +0200 Subject: [PATCH] update to admin/bulk/search API, add missing HTTP actions get index, search scroll, clear scroll --- .../java/org/xbib/elx/api/AdminClient.java | 187 +++++ .../org/xbib/elx/api/AdminClientProvider.java | 7 + .../java/org/xbib/elx/api/BasicClient.java | 62 ++ .../java/org/xbib/elx/api/BulkClient.java | 227 ++++++ .../org/xbib/elx/api/BulkClientProvider.java | 7 + .../java/org/xbib/elx/api/BulkMetric.java | 3 + .../java/org/xbib/elx/api/ExtendedClient.java | 487 ------------- .../xbib/elx/api/ExtendedClientProvider.java | 7 - .../org/xbib/elx/api/IndexDefinition.java | 14 +- .../org/xbib/elx/api/IndexPruneResult.java | 6 +- .../org/xbib/elx/api/IndexShiftResult.java | 6 +- .../java/org/xbib/elx/api/ReadClient.java | 25 - .../org/xbib/elx/api/ReadClientProvider.java | 6 - .../java/org/xbib/elx/api/SearchClient.java | 27 + .../xbib/elx/api/SearchClientProvider.java | 7 + ...edClient.java => AbstractAdminClient.java} | 660 ++++-------------- .../xbib/elx/common/AbstractBasicClient.java | 217 ++++++ .../xbib/elx/common/AbstractBulkClient.java | 246 +++++++ .../xbib/elx/common/AbstractSearchClient.java | 158 +++++ .../org/xbib/elx/common/ClientBuilder.java | 94 ++- .../elx/common/DefaultBulkController.java | 20 +- .../xbib/elx/common/DefaultBulkMetric.java | 5 + .../elx/common/DefaultIndexDefinition.java | 40 +- .../org/xbib/elx/common/MockAdminClient.java | 59 ++ .../elx/common/MockAdminClientProvider.java | 10 + .../org/xbib/elx/common/MockBulkClient.java | 103 +++ .../elx/common/MockBulkClientProvider.java | 11 + .../xbib/elx/common/MockExtendedClient.java | 129 ---- .../common/MockExtendedClientProvider.java | 10 - .../org/xbib/elx/common/MockSearchClient.java | 44 ++ .../elx/common/MockSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../common/test/MockClientProviderTest.java | 41 ++ .../test/MockExtendedClientProviderTest.java | 19 - .../resources/{log4j2.xml => log4j2-test.xml} | 0 .../elx/http/ExtendedHttpClientProvider.java | 12 - .../org/xbib/elx/http/HttpActionContext.java | 6 +- .../org/xbib/elx/http/HttpAdminClient.java | 55 ++ .../elx/http/HttpAdminClientProvider.java | 11 + .../org/xbib/elx/http/HttpBulkClient.java | 55 ++ .../xbib/elx/http/HttpBulkClientProvider.java | 11 + ...dHttpClient.java => HttpClientHelper.java} | 34 +- .../org/xbib/elx/http/HttpSearchClient.java | 55 ++ .../elx/http/HttpSearchClientProvider.java | 11 + .../admin/indices/get/HtppGetIndexAction.java | 35 + .../action/search/HttpClearScrollAction.java | 35 + .../http/action/search/HttpSearchAction.java | 6 +- .../action/search/HttpSearchScrollAction.java | 35 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../services/org.xbib.elx.http.HttpAction | 3 + .../xbib/elx/http/test/BulkClientTest.java | 182 +++++ .../org/xbib/elx/http/test/ClientTest.java | 244 ------- .../xbib/elx/http/test/DuplicateIDTest.java | 34 +- .../xbib/elx/http/test/IndexPruneTest.java | 55 +- .../xbib/elx/http/test/IndexShiftTest.java | 67 +- .../org/xbib/elx/http/test/SearchTest.java | 82 +++ .../org/xbib/elx/http/test/SmokeTest.java | 78 +-- .../resources/{log4j2.xml => log4j2-test.xml} | 0 .../org/xbib/elx/node/ExtendedNodeClient.java | 69 -- .../elx/node/ExtendedNodeClientProvider.java | 10 - .../org/xbib/elx/node/NodeAdminClient.java | 25 + .../elx/node/NodeAdminClientProvider.java | 11 + .../org/xbib/elx/node/NodeBulkClient.java | 25 + .../xbib/elx/node/NodeBulkClientProvider.java | 11 + .../org/xbib/elx/node/NodeClientHelper.java | 98 +++ .../org/xbib/elx/node/NodeSearchClient.java | 25 + .../elx/node/NodeSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../xbib/elx/node/test/BulkClientTest.java | 180 +++++ .../org/xbib/elx/node/test/ClientTest.java | 236 ------- .../xbib/elx/node/test/DuplicateIDTest.java | 54 +- .../xbib/elx/node/test/IndexPruneTest.java | 54 +- .../xbib/elx/node/test/IndexShiftTest.java | 66 +- .../org/xbib/elx/node/test/SearchTest.java | 82 +++ .../org/xbib/elx/node/test/SmokeTest.java | 79 +-- .../org/xbib/elx/node/test/TestExtension.java | 1 + .../resources/{log4j2.xml => log4j2-test.xml} | 0 .../ExtendedTransportClientProvider.java | 11 - .../elx/transport/TransportAdminClient.java | 36 + .../TransportAdminClientProvider.java | 11 + .../elx/transport/TransportBulkClient.java | 35 + .../TransportBulkClientProvider.java | 11 + ...Client.java => TransportClientHelper.java} | 52 +- .../elx/transport/TransportSearchClient.java | 35 + .../TransportSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../elx/transport/test/BulkClientTest.java | 190 +++++ .../xbib/elx/transport/test/ClientTest.java | 247 ------- .../elx/transport/test/DuplicateIDTest.java | 53 +- .../elx/transport/test/IndexPruneTest.java | 53 +- .../elx/transport/test/IndexShiftTest.java | 58 +- .../xbib/elx/transport/test/SearchTest.java | 82 +++ .../xbib/elx/transport/test/SmokeTest.java | 75 +- .../resources/{log4j2.xml => log4j2-test.xml} | 0 106 files changed, 3509 insertions(+), 2565 deletions(-) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/AdminClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BasicClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/ReadClient.java delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/SearchClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java rename elx-common/src/main/java/org/xbib/elx/common/{AbstractExtendedClient.java => AbstractAdminClient.java} (55%) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java delete mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java delete mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java create mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider create mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/MockClientProviderTest.java delete mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java rename elx-common/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) delete mode 100644 elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpAdminClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java rename elx-http/src/main/java/org/xbib/elx/http/{ExtendedHttpClient.java => HttpClientHelper.java} (84%) create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/get/HtppGetIndexAction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/search/HttpClearScrollAction.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchScrollAction.java create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider create mode 100644 elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java delete mode 100644 elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java create mode 100644 elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java rename elx-http/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) delete mode 100644 elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java delete mode 100644 elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeClientHelper.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java rename elx-node/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) delete mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java rename elx-transport/src/main/java/org/xbib/elx/transport/{ExtendedTransportClient.java => TransportClientHelper.java} (78%) create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java delete mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java rename elx-transport/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java new file mode 100644 index 0000000..d43a89a --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java @@ -0,0 +1,187 @@ +package org.xbib.elx.api; + +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Interface for extended managing and indexing methods of an Elasticsearch client. + */ +public interface AdminClient extends BasicClient { + + /** + * Build index definition from settings. + * + * @param index the index name + * @param settings the settings for the index + * @return index definition + * @throws IOException if settings/mapping URL is invalid/malformed + */ + IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; + + Map getMapping(String index) throws IOException; + + Map getMapping(String index, String type) throws IOException; + + /** + * Delete an index. + * @param indexDefinition the index definition + * @return this + */ + AdminClient deleteIndex(IndexDefinition indexDefinition); + + /** + * Delete an index. + * + * @param index index + * @return this + */ + AdminClient deleteIndex(String index); + + + /** + * Update replica level. + * @param indexDefinition the index definition + * @param level the replica level + * @return this + * @throws IOException if replica setting could not be updated + */ + AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; + + /** + * Update replica level. + * + * @param index index + * @param level the replica level + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return this + * @throws IOException if replica setting could not be updated + */ + AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; + + /** + * Get replica level. + * @param indexDefinition the index name + * @return the replica level of the index + */ + int getReplicaLevel(IndexDefinition indexDefinition); + + /** + * Get replica level. + * @param index the index name + * @return the replica level of the index + */ + int getReplicaLevel(String index); + + /** + * Force segment merge of an index. + * @param indexDefinition the index definition + * @return this + */ + boolean forceMerge(IndexDefinition indexDefinition); + + /** + * Force segment merge of an index. + * @param index the index + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return this + */ + boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); + + /** + * Resolve alias. + * + * @param alias the alias + * @return this index name behind the alias or the alias if there is no index + */ + String resolveAlias(String alias); + + /** + * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index + * name. + * + * @param alias the alias + * @return the most recent index name pointing to the alias + */ + String resolveMostRecentIndex(String alias); + + /** + * Get all index aliases. + * @param index the index + * @return map of index aliases + */ + Map getAliases(String index); + + /** + * Shift from one index to another. + * @param indexDefinition the index definition + * @param additionalAliases new aliases + * @return this + */ + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); + + /** + * Shift from one index to another. + * @param indexDefinition the index definition + * @param additionalAliases new aliases + * @param indexAliasAdder method to add aliases + * @return this + */ + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, + IndexAliasAdder indexAliasAdder); + + /** + * Shift from one index to another. + * @param index the index name + * @param fullIndexName the index name with timestamp + * @param additionalAliases a list of names that should be set as index aliases + * @return this + */ + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); + + /** + * Shift from one index to another. + * @param index the index name + * @param fullIndexName the index name with timestamp + * @param additionalAliases a list of names that should be set as index aliases + * @param adder an adder method to create alias term queries + * @return this + */ + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, + IndexAliasAdder adder); + + /** + * Prune index. + * @param indexDefinition the index definition + * @return the index prune result + */ + IndexPruneResult pruneIndex(IndexDefinition indexDefinition); + + /** + * Apply retention policy to prune indices. All indices before delta should be deleted, + * but the number of mintokeep indices must be kept. + * + * @param index index name + * @param fullIndexName index name with timestamp + * @param delta timestamp delta (for index timestamps) + * @param mintokeep minimum number of indices to keep + * @param perform true if pruning should be executed, false if not + * @return the index prune result + */ + IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); + + /** + * Find the timestamp of the most recently indexed document in the index. + * + * @param index the index name + * @param timestampfieldname the timestamp field name + * @return millis UTC millis of the most recent document + * @throws IOException if most rcent document can not be found + */ + Long mostRecentDocument(String index, String timestampfieldname) throws IOException; +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java new file mode 100644 index 0000000..954efd5 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface AdminClientProvider { + + C getClient(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BasicClient.java b/elx-api/src/main/java/org/xbib/elx/api/BasicClient.java new file mode 100644 index 0000000..9b9b4b5 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BasicClient.java @@ -0,0 +1,62 @@ +package org.xbib.elx.api; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import java.io.Closeable; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +public interface BasicClient extends Closeable { + + /** + * Set an Elasticsearch client to extend from it. May be null for TransportClient. + * @param client the Elasticsearch client + */ + void setClient(ElasticsearchClient client); + + /** + * Return Elasticsearch client. + * + * @return Elasticsearch client + */ + ElasticsearchClient getClient(); + + /** + * Initiative the extended client, the bulk metric and bulk controller, + * creates instances and connect to cluster, if required. + * + * @param settings settings + * @throws IOException if init fails + */ + void init(Settings settings) throws IOException; + + /** + * Get cluster name. + * @return the cluster name + */ + String getClusterName(); + + /** + * Get current health color. + * + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return the cluster health color + */ + String getHealthColor(long maxWaitTime, TimeUnit timeUnit); + + /** + * Wait for cluster being healthy. + * + * @param healthColor cluster health color to wait for + * @param maxWaitTime time value + * @param timeUnit time unit + */ + void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); + + void waitForShards(long maxWaitTime, TimeUnit timeUnit); + + long getSearchableDocs(String index); + + boolean isIndexExists(String index); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java new file mode 100644 index 0000000..e7ffc59 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java @@ -0,0 +1,227 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.Flushable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public interface BulkClient extends BasicClient, Flushable { + + /** + * Get bulk metric. + * @return the bulk metric + */ + BulkMetric getBulkMetric(); + + /** + * Get buulk control. + * @return the bulk control + */ + BulkController getBulkController(); + + + /** + * Create a new index. + * + * @param index index + * @throws IOException if new index creation fails + */ + void newIndex(String index) throws IOException; + + /** + * Create a new index. + * @param indexDefinition the index definition + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(IndexDefinition indexDefinition) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @throws IOException if settings is invalid or index creation fails + */ + void newIndex(String index, Settings settings) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(String index, Settings settings, Map mapping) throws IOException; + + /** + * Add index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. + * + * @param index the index + * @param id the id + * @param create true if document must be created + * @param source the source + * @return this + */ + BulkClient index(String index, String id, boolean create, BytesReference source); + + /** + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. + * + * @param index the index + * @param id the id + * @param create true if document is to be created, false otherwise + * @param source the source + * @return this client methods + */ + BulkClient index(String index, String id, boolean create, String source); + + /** + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param indexRequest the index request to add + * @return this + */ + BulkClient index(IndexRequest indexRequest); + + /** + * Delete request. + * + * @param index the index + * @param id the id + * @return this + */ + BulkClient delete(String index, String id); + + /** + * Delete request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param deleteRequest the delete request to add + * @return this + */ + BulkClient delete(DeleteRequest deleteRequest); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param index the index + * @param id the id + * @param source the source + * @return this + */ + BulkClient update(String index, String id, BytesReference source); + + /** + * Update document. Use with precaution! Does not work in all cases. + * + * @param index the index + * @param id the id + * @param source the source + * @return this + */ + BulkClient update(String index, String id, String source); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param updateRequest the update request to add + * @return this + */ + BulkClient update(UpdateRequest updateRequest); + + /** + * Start bulk mode for indexes. + * @param indexDefinition index definition + * @throws IOException if bulk could not be started + */ + void startBulk(IndexDefinition indexDefinition) throws IOException; + + /** + * Start bulk mode. + * + * @param index index + * @param startRefreshIntervalSeconds refresh interval before bulk + * @param stopRefreshIntervalSeconds refresh interval after bulk + * @throws IOException if bulk could not be started + */ + void startBulk(String index, long startRefreshIntervalSeconds, + long stopRefreshIntervalSeconds) throws IOException; + + + /** + * Stop bulk mode. + * + * @param indexDefinition index definition + * @throws IOException if bulk could not be startet + */ + void stopBulk(IndexDefinition indexDefinition) throws IOException; + + /** + * Stops bulk mode. + * + * @param index index + * @param timeout maximum wait time + * @param timeUnit time unit for timeout + * @throws IOException if bulk could not be stopped + */ + void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; + + /** + * Wait for all outstanding bulk responses. + * + * @param timeout maximum wait time + * @param timeUnit unit of timeout value + * @return true if wait succeeded, false if wait timed out + */ + boolean waitForResponses(long timeout, TimeUnit timeUnit); + + /** + * Update index setting. + * @param index the index + * @param key the key of the value to be updated + * @param value the new value + * @param timeout timeout + * @param timeUnit time unit + * @throws IOException if update index setting failed + */ + void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; + + /** + * Refresh the index. + * + * @param index index + */ + void refreshIndex(String index); + + /** + * Flush the index. The cluster clears cache and completes indexing. + * + * @param index index + */ + void flushIndex(String index); + +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java new file mode 100644 index 0000000..0f013e2 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface BulkClientProvider { + + C getClient(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java index 7e84376..af825e5 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java @@ -1,5 +1,6 @@ package org.xbib.elx.api; +import org.elasticsearch.common.settings.Settings; import org.xbib.metrics.api.Count; import org.xbib.metrics.api.Metered; @@ -7,6 +8,8 @@ import java.io.Closeable; public interface BulkMetric extends Closeable { + void init(Settings settings); + Metered getTotalIngest(); Count getTotalIngestSizeInBytes(); diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java deleted file mode 100644 index e1e5ac2..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ /dev/null @@ -1,487 +0,0 @@ -package org.xbib.elx.api; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Interface for extended managing and indexing methods of an Elasticsearch client. - */ -public interface ExtendedClient extends Flushable, Closeable { - - /** - * Set an Elasticsearch client to extend from it. May be null for TransportClient. - * @param client client - * @return this client - */ - ExtendedClient setClient(ElasticsearchClient client); - - /** - * Return Elasticsearch client. - * - * @return Elasticsearch client - */ - ElasticsearchClient getClient(); - - /** - * Get bulk control. - * @return the bulk control - */ - BulkController getBulkController(); - - /** - * Initiative the extended client, the bulk metric and bulk controller, - * creates instances and connect to cluster, if required. - * - * @param settings settings - * @return this client - * @throws IOException if init fails - */ - ExtendedClient init(Settings settings) throws IOException; - - /** - * Build index definition from settings. - * - * @param index the index name - * @param settings the settings for the index - * @return index definition - * @throws IOException if settings/mapping URL is invalid/malformed - */ - IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; - - /** - * Add index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when limits are exceeded. - * - * @param index the index - * @param id the id - * @param create true if document must be created - * @param source the source - * @return this - */ - ExtendedClient index(String index, String id, boolean create, BytesReference source); - - /** - * Index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when limits are exceeded. - * - * @param index the index - * @param id the id - * @param create true if document is to be created, false otherwise - * @param source the source - * @return this client methods - */ - ExtendedClient index(String index, String id, boolean create, String source); - - /** - * Index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param indexRequest the index request to add - * @return this - */ - ExtendedClient index(IndexRequest indexRequest); - - /** - * Delete request. - * - * @param index the index - * @param id the id - * @return this - */ - ExtendedClient delete(String index, String id); - - /** - * Delete request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param deleteRequest the delete request to add - * @return this - */ - ExtendedClient delete(DeleteRequest deleteRequest); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized. - * - * @param index the index - * @param id the id - * @param source the source - * @return this - * @throws IOException if update fails - */ - ExtendedClient update(String index, String id, BytesReference source) throws IOException; - - /** - * Update document. Use with precaution! Does not work in all cases. - * - * @param index the index - * @param id the id - * @param source the source - * @return this - */ - ExtendedClient update(String index, String id, String source); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized. - * - * @param updateRequest the update request to add - * @return this - */ - ExtendedClient update(UpdateRequest updateRequest); - - /** - * Create a new index. - * - * @param index index - * @return this - * @throws IOException if new index creation fails - */ - ExtendedClient newIndex(String index) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @return this - * @throws IOException if settings is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; - - /** - * Create a new index. - * @param indexDefinition the index definition - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException; - - /** - * Delete an index. - * @param indexDefinition the index definition - * @return this - */ - ExtendedClient deleteIndex(IndexDefinition indexDefinition); - - /** - * Delete an index. - * - * @param index index - * @return this - */ - ExtendedClient deleteIndex(String index); - - /** - * Start bulk mode for indexes. - * @param indexDefinition index definition - * @return this - * @throws IOException if bulk could not be started - */ - ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException; - - /** - * Start bulk mode. - * - * @param index index - * @param startRefreshIntervalSeconds refresh interval before bulk - * @param stopRefreshIntervalSeconds refresh interval after bulk - * @return this - * @throws IOException if bulk could not be started - */ - ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, - long stopRefreshIntervalSeconds) throws IOException; - - /** - * Stop bulk mode. - * - * @param indexDefinition index definition - * @return this - * @throws IOException if bulk could not be startet - */ - ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException; - - /** - * Stops bulk mode. - * - * @param index index - * @param timeout maximum wait time - * @param timeUnit time unit for timeout - * @return this - * @throws IOException if bulk could not be stopped - */ - ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; - - /** - * Update replica level. - * @param indexDefinition the index definition - * @param level the replica level - * @return this - * @throws IOException if replica setting could not be updated - */ - ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; - - /** - * Update replica level. - * - * @param index index - * @param level the replica level - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return this - * @throws IOException if replica setting could not be updated - */ - ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; - - /** - * Get replica level. - * @param indexDefinition the index name - * @return the replica level of the index - */ - int getReplicaLevel(IndexDefinition indexDefinition); - - /** - * Get replica level. - * @param index the index name - * @return the replica level of the index - */ - int getReplicaLevel(String index); - - /** - * Refresh the index. - * - * @param index index - * @return this - */ - ExtendedClient refreshIndex(String index); - - /** - * Flush the index. The cluster clears cache and completes indexing. - * - * @param index index - * @return this - */ - ExtendedClient flushIndex(String index); - - /** - * Force segment merge of an index. - * @param indexDefinition the index definition - * @return this - */ - boolean forceMerge(IndexDefinition indexDefinition); - - /** - * Force segment merge of an index. - * @param index the index - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return this - */ - boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); - - /** - * Wait for all outstanding bulk responses. - * - * @param timeout maximum wait time - * @param timeUnit unit of timeout value - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForResponses(long timeout, TimeUnit timeUnit); - - /** - * Wait for cluster being healthy. - * - * @param healthColor cluster health color to wait for - * @param maxWaitTime time value - * @param timeUnit time unit - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); - - /** - * Get current health color. - * - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return the cluster health color - */ - String getHealthColor(long maxWaitTime, TimeUnit timeUnit); - - /** - * Wait for index recovery (after replica change). - * - * @param index index - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit); - - /** - * Update index setting. - * @param index the index - * @param key the key of the value to be updated - * @param value the new value - * @param timeout timeout - * @param timeUnit time unit - * @throws IOException if update index setting failed - */ - void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; - - /** - * Resolve alias. - * - * @param alias the alias - * @return this index name behind the alias or the alias if there is no index - */ - String resolveAlias(String alias); - - /** - * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index - * name. - * - * @param alias the alias - * @return the most recent index name pointing to the alias - */ - String resolveMostRecentIndex(String alias); - - /** - * Get all index aliases. - * @param index the index - * @return map of index aliases - */ - Map getAliases(String index); - - /** - * Shift from one index to another. - * @param indexDefinition the index definition - * @param additionalAliases new aliases - * @return this - */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); - - /** - * Shift from one index to another. - * @param indexDefinition the index definition - * @param additionalAliases new aliases - * @param indexAliasAdder method to add aliases - * @return this - */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, - IndexAliasAdder indexAliasAdder); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @param adder an adder method to create alias term queries - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, - IndexAliasAdder adder); - - /** - * Prune index. - * @param indexDefinition the index definition - * @return the index prune result - */ - IndexPruneResult pruneIndex(IndexDefinition indexDefinition); - - /** - * Apply retention policy to prune indices. All indices before delta should be deleted, - * but the number of mintokeep indices must be kept. - * - * @param index index name - * @param fullIndexName index name with timestamp - * @param delta timestamp delta (for index timestamps) - * @param mintokeep minimum number of indices to keep - * @param perform true if pruning should be executed, false if not - * @return the index prune result - */ - IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); - - /** - * Find the timestamp of the most recently indexed document in the index. - * - * @param index the index name - * @param timestampfieldname the timestamp field name - * @return millis UTC millis of the most recent document - * @throws IOException if most rcent document can not be found - */ - Long mostRecentDocument(String index, String timestampfieldname) throws IOException; - - /** - * Get cluster name. - * @return the cluster name - */ - String getClusterName(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java deleted file mode 100644 index 2a8904a..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.xbib.elx.api; - -@FunctionalInterface -public interface ExtendedClientProvider { - - C getExtendedClient(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 49544a7..1c77ccb 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -1,7 +1,5 @@ package org.xbib.elx.api; -import java.net.MalformedURLException; -import java.net.URL; import java.util.concurrent.TimeUnit; public interface IndexDefinition { @@ -14,17 +12,13 @@ public interface IndexDefinition { String getFullIndexName(); - IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException; + IndexDefinition setSettings(String settings); - IndexDefinition setSettingsUrl(URL settingsUrl); + String getSettings(); - URL getSettingsUrl(); + IndexDefinition setMappings(String mappings); - IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException; - - IndexDefinition setMappingsUrl(URL mappingsUrl); - - URL getMappingsUrl(); + String getMappings(); IndexDefinition setDateTimePattern(String timeWindow); diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java index 0c118f8..cef89b8 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java @@ -1,6 +1,6 @@ package org.xbib.elx.api; -import java.util.List; +import java.util.Collection; public interface IndexPruneResult { @@ -8,9 +8,9 @@ public interface IndexPruneResult { State getState(); - List getCandidateIndices(); + Collection getCandidateIndices(); - List getDeletedIndices(); + Collection getDeletedIndices(); boolean isAcknowledged(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java index 02a2e8c..6ef58d7 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java @@ -1,10 +1,10 @@ package org.xbib.elx.api; -import java.util.List; +import java.util.Collection; public interface IndexShiftResult { - List getMovedAliases(); + Collection getMovedAliases(); - List getNewAliases(); + Collection getNewAliases(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java deleted file mode 100644 index 2ecb857..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.xbib.elx.api; - -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; - -public interface ReadClient { - - ActionFuture get(GetRequest getRequest); - - void get(GetRequest request, ActionListener listener); - - ActionFuture multiGet(MultiGetRequest request); - - void multiGet(MultiGetRequest request, ActionListener listener); - - ActionFuture search(SearchRequest request); - - void search(SearchRequest request, ActionListener listener); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java deleted file mode 100644 index bc0eb16..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.xbib.elx.api; - -public interface ReadClientProvider { - - C getReadClient(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java b/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java new file mode 100644 index 0000000..3a04949 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java @@ -0,0 +1,27 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Stream; + +public interface SearchClient extends BasicClient { + + Optional get(Consumer getRequestBuilder); + + Optional multiGet(Consumer multiGetRequestBuilder); + + Optional search(Consumer searchRequestBuilder); + + Stream search(Consumer searchRequestBuilder, + TimeValue scrollTime, int scrollSize); + + Stream getIds(Consumer queryBuilder); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java new file mode 100644 index 0000000..23bfb07 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface SearchClientProvider { + + C getClient(); +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java similarity index 55% rename from elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java rename to elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index a4a54d4..6219839 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -2,14 +2,8 @@ package org.xbib.elx.common; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.search.TotalHits; -import org.elasticsearch.ElasticsearchTimeoutException; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; @@ -18,13 +12,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.flush.FlushAction; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexAction; @@ -32,36 +21,31 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -69,8 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; -import org.xbib.elx.api.BulkController; -import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.AdminClient; import org.xbib.elx.api.IndexAliasAdder; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexPruneResult; @@ -79,13 +62,15 @@ import org.xbib.elx.api.IndexShiftResult; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -98,22 +83,21 @@ import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -public abstract class AbstractExtendedClient implements ExtendedClient { +public abstract class AbstractAdminClient extends AbstractBasicClient implements AdminClient { - private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); + private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName()); - private ElasticsearchClient client; - - private BulkController bulkController; - - private final AtomicBoolean closed; + /** + * The one and only index type name used in the extended client. + * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". + */ + private static final String TYPE_NAME = "doc"; private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { @Override @@ -149,363 +133,54 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } }; - protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; - - protected abstract void closeClient() throws IOException; - - protected AbstractExtendedClient() { - closed = new AtomicBoolean(false); + @Override + public Map getMapping(String index) throws IOException { + return getMapping(index, TYPE_NAME); } @Override - public AbstractExtendedClient setClient(ElasticsearchClient client) { - this.client = client; - return this; + public Map getMapping(String index, String mapping) throws IOException { + GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) + .setIndices(index) + .setTypes(mapping); + GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet(); + logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap()); + return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap(); } @Override - public ElasticsearchClient getClient() { - return client; - } - - @Override - public BulkController getBulkController() { - return bulkController; - } - - @Override - public AbstractExtendedClient init(Settings settings) throws IOException { - logger.info("initializing with settings = " + settings.toDelimitedString(',')); - if (client == null) { - client = createClient(settings); - } - if (bulkController == null) { - bulkController = new DefaultBulkController(this); - bulkController.init(settings); - } - return this; - } - - @Override - public void flush() throws IOException { - if (bulkController != null) { - bulkController.flush(); - } - } - - @Override - public void close() throws IOException { - ensureClient(); - if (closed.compareAndSet(false, true)) { - if (bulkController != null) { - logger.info("closing bulk controller"); - bulkController.close(); - } - closeClient(); - } - } - - @Override - public String getClusterName() { - ensureClient(); - try { - ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); - ClusterStateResponse clusterStateResponse = - client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - return clusterStateResponse.getClusterName().value(); - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - @Override - public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException { - ensureClient(); - waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); - URL indexSettings = indexDefinition.getSettingsUrl(); - if (indexSettings == null) { - logger.warn("warning while creating index '{}', no settings/mappings", - indexDefinition.getFullIndexName()); - newIndex(indexDefinition.getFullIndexName()); - return this; - } - URL indexMappings = indexDefinition.getMappingsUrl(); - if (indexMappings == null) { - logger.warn("warning while creating index '{}', no mappings", - indexDefinition.getFullIndexName()); - newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null); - return this; - } - try (InputStream indexSettingsInput = indexSettings.openStream(); - InputStream indexMappingsInput = indexMappings.openStream()) { - newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput); - } catch (IOException e) { - if (indexDefinition.ignoreErrors()) { - logger.warn(e.getMessage(), e); - logger.warn("warning while creating index '{}' with settings at {} and mappings at {}", - indexDefinition.getFullIndexName(), indexSettings, indexMappings); - } else { - logger.error("error while creating index '{}' with settings at {} and mappings at {}", - indexDefinition.getFullIndexName(), indexSettings, indexMappings); - throw new IOException(e); - } - } - return this; - } - - @Override - public ExtendedClient newIndex(String index) throws IOException { - return newIndex(index, Settings.EMPTY, (Map) null); - } - - @Override - public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException { - return newIndex(index, - Settings.builder().loadFromStream(".json", settings, true).build(), - mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings) throws IOException { - return newIndex(index, settings, (Map) null); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException { - return newIndex(index, settings, - mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException { - ensureClient(); - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); - if (settings != null) { - createIndexRequest.settings(settings); - } - if (mapping != null) { - createIndexRequest.mapping("_doc", mapping); - } - CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - if (createIndexResponse.isAcknowledged()) { - return this; - } - throw new IllegalStateException("index creation not acknowledged"); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { - ensureClient(); - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); - if (settings != null) { - createIndexRequest.settings(settings); - } - if (mapping != null) { - createIndexRequest.mapping("_doc", mapping); - } - CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - if (createIndexResponse.isAcknowledged()) { - return this; - } - throw new IllegalStateException("index creation not acknowledged"); - } - - @Override - public ExtendedClient deleteIndex(IndexDefinition indexDefinition) { + public AdminClient deleteIndex(IndexDefinition indexDefinition) { return deleteIndex(indexDefinition.getFullIndexName()); } @Override - public ExtendedClient deleteIndex(String index) { - ensureClient(); + public AdminClient deleteIndex(String index) { + ensureClientIsPresent(); if (index == null) { logger.warn("no index name given to delete index"); return this; } DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index); client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); + waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); + waitForShards(30L, TimeUnit.SECONDS); return this; } @Override - public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException { - startBulk(indexDefinition.getFullIndexName(), -1, 1); - return this; - } - - @Override - public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) - throws IOException { - if (bulkController != null) { - ensureClient(); - bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); - } - return this; - } - - @Override - public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException { - if (bulkController != null) { - ensureClient(); - bulkController.stopBulkMode(indexDefinition); - } - return this; - } - - @Override - public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { - if (bulkController != null) { - ensureClient(); - bulkController.stopBulkMode(index, timeout, timeUnit); - } - return this; - } - - @Override - public ExtendedClient index(String index, String id, boolean create, String source) { - return index(new IndexRequest().index(index).id(id).create(create) - .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); - } - - @Override - public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest().index(index).id(id).create(create) - .source(source, XContentType.JSON)); - } - - @Override - public ExtendedClient index(IndexRequest indexRequest) { - ensureClient(); - bulkController.bulkIndex(indexRequest); - return this; - } - - @Override - public ExtendedClient delete(String index, String id) { - return delete(new DeleteRequest().index(index).id(id)); - } - - @Override - public ExtendedClient delete(DeleteRequest deleteRequest) { - ensureClient(); - bulkController.bulkDelete(deleteRequest); - return this; - } - - @Override - public ExtendedClient update(String index, String id, BytesReference source) { - return update(new UpdateRequest().index(index).id(id) - .doc(source, XContentType.JSON)); - } - - @Override - public ExtendedClient update(String index, String id, String source) { - return update(new UpdateRequest().index(index).id(id) - .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); - } - - @Override - public ExtendedClient update(UpdateRequest updateRequest) { - ensureClient(); - bulkController.bulkUpdate(updateRequest); - return this; - } - - @Override - public boolean waitForResponses(long timeout, TimeUnit timeUnit) { - ensureClient(); - return bulkController.waitForBulkResponses(timeout, timeUnit); - } - - @Override - public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - ensureClient(); - ensureIndexGiven(index); - GetSettingsRequest settingsRequest = new GetSettingsRequest(); - settingsRequest.indices(index); - GetSettingsResponse settingsResponse = client.execute(GetSettingsAction.INSTANCE, settingsRequest).actionGet(); - int shards = settingsResponse.getIndexToSettings().get(index).getAsInt("index.number_of_shards", -1); - if (shards > 0) { - TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() - .indices(index) - .waitForActiveShards(shards) - .timeout(timeout); - ClusterHealthResponse healthResponse = - client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - logger.warn("timeout waiting for recovery"); - return false; - } - } - return true; - } - - @Override - public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { - ensureClient(); - ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); - TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); - return false; - } - return true; - } - - @Override - public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { - ensureClient(); - try { - TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(timeout)).actionGet(); - ClusterHealthStatus status = healthResponse.getStatus(); - return status.name(); - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - @Override - public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException { + public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException { return updateReplicaLevel(indexDefinition.getFullIndexName(), level, indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit()); } @Override - public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { - waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations - if (level > 0) { - updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); - waitForRecovery(index, maxWaitTime, timeUnit); + public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { + if (level < 1) { + logger.warn("invalid replica level"); + return this; } + updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); + waitForShards(maxWaitTime, timeUnit); return this; } @@ -528,30 +203,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return replica; } - @Override - public ExtendedClient flushIndex(String index) { - if (index != null) { - ensureClient(); - client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); - } - return this; - } - - @Override - public ExtendedClient refreshIndex(String index) { - if (index != null) { - ensureClient(); - client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); - } - return this; - } - @Override public String resolveMostRecentIndex(String alias) { + ensureClientIsPresent(); if (alias == null) { return null; } - ensureClient(); GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias); GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); @@ -576,7 +233,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String resolveAlias(String alias) { - ensureClient(); + ensureClientIsPresent(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); clusterStateRequest.blocks(false); clusterStateRequest.metaData(true); @@ -618,7 +275,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, IndexAliasAdder adder) { - ensureClient(); + ensureClientIsPresent(); if (index == null) { return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to } @@ -644,9 +301,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String alias = entry.getKey(); String filter = entry.getValue(); indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() - .indices(oldIndex).alias(alias)); + .index(oldIndex).alias(alias)); if (filter != null) { - indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() .index(fullIndexName).alias(alias).filter(filter)); } else { indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() @@ -670,7 +327,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } else { String filter = oldAliasMap.get(additionalAlias); indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() - .indices(oldIndex).alias(additionalAlias)); + .index(oldIndex).alias(additionalAlias)); if (filter != null) { indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(fullIndexName).alias(additionalAlias).filter(filter)); @@ -686,8 +343,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString()); AcknowledgedResponse indicesAliasesResponse = client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); - logger.debug("response isAcknowledged = {} isFragment = {}", - indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment()); + logger.debug("response isAcknowledged = {}", + indicesAliasesResponse.isAcknowledged()); } return new SuccessIndexShiftResult(moveAliases, newAliases); } @@ -706,11 +363,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (index.equals(fullIndexName)) { return EMPTY_INDEX_PRUNE_RESULT; } - ensureClient(); + ensureClientIsPresent(); GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length); + logger.info("{} indices", getIndexResponse.getIndices().length); List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); @@ -746,29 +403,20 @@ public abstract class AbstractExtendedClient implements ExtendedClient { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); - if( response.isAcknowledged()) { - logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s)); - waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - return new SuccessPruneResult(candidateIndices, indicesToDelete, response); - } else { - logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s)); - return new FailPruneResult(candidateIndices, indicesToDelete, response); - } + return new SuccessPruneResult(candidateIndices, indicesToDelete, response); } @Override public Long mostRecentDocument(String index, String timestampfieldname) { - ensureClient(); - SortBuilder sort = SortBuilders - .fieldSort(timestampfieldname) - .order(SortOrder.DESC); - SearchSourceBuilder builder = new SearchSourceBuilder() - .sort(sort) - .storedField(timestampfieldname) - .size(1); - SearchRequest searchRequest = new SearchRequest() - .indices(index) - .source(builder); + ensureClientIsPresent(); + SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.sort(sort); + builder.storedField(timestampfieldname); + builder.size(1); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.source(builder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); if (searchResponse.getHits().getHits().length == 1) { @@ -813,7 +461,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException { - boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); + boolean isEnabled = settings.getAsBoolean("enabled", false); String indexName = settings.get("name", index); String fullIndexName; String dateTimePattern = settings.get("dateTimePattern"); @@ -833,8 +481,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .setEnabled(isEnabled) .setIndex(indexName) .setFullIndexName(fullIndexName) - .setSettingsUrl(settings.get("settings")) - .setMappingsUrl(settings.get("mapping")) + .setSettings(findSettingsFrom(settings.get("settings"))) + .setMappings(findMappingsFrom(settings.get("mapping"))) .setDateTimePattern(dateTimePattern) .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) .setShift(settings.getAsBoolean("shift", true)) @@ -847,16 +495,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { - ensureClient(); + ensureClientIsPresent(); if (index == null) { throw new IOException("no index name given"); } - if (key == null) { - throw new IOException("no key given"); - } - if (value == null) { - throw new IOException("no value given"); - } Settings.Builder updateSettingsBuilder = Settings.builder(); updateSettingsBuilder.put(key, value.toString()); UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) @@ -864,18 +506,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient { client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); } - private void ensureClient() { - if (this instanceof MockExtendedClient) { - return; + private static String findSettingsFrom(String string) throws IOException { + if (string == null) { + return null; } - if (client == null) { - throw new IllegalStateException("no client"); + try { + URL url = new URL(string); + try (InputStream inputStream = url.openStream()) { + Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build(); + XContentBuilder builder = JsonXContent.contentBuilder(); + settings.toXContent(builder, ToXContent.EMPTY_PARAMS); + return Strings.toString(builder); + } + } catch (MalformedURLException e) { + return string; } } - private void ensureIndexGiven(String index) { - if (index == null) { - throw new IllegalArgumentException("no index given"); + private static String findMappingsFrom(String string) throws IOException { + if (string == null) { + return null; + } + try { + URL url = new URL(string); + try (InputStream inputStream = url.openStream()) { + if (string.endsWith(".json")) { + Map mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered(); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject().map(mappings).endObject(); + return Strings.toString(builder); + } + if (string.endsWith(".yml") || string.endsWith(".yaml")) { + Map mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered(); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject().map(mappings).endObject(); + return Strings.toString(builder); + } + } + return string; + } catch (MalformedInputException e) { + return string; } } @@ -896,7 +568,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } public void checkMapping(String index) { - ensureClient(); + ensureClientIsPresent(); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index); GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); ImmutableOpenMap> map = getMappingsResponse.getMappings(); @@ -912,25 +584,25 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { try { - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices(index) - .source(builder); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) + .setIndices(index) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0) + .setTrackTotalHits(true); SearchResponse searchResponse = - client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); - TotalHits total = searchResponse.getHits().getTotalHits(); - if (total.value > 0L) { + searchRequestBuilder.execute().actionGet(); + long total = searchResponse.getHits().getTotalHits().value; + if (total > 0L) { Map fields = new TreeMap<>(); Map root = mappingMetaData.getSourceAsMap(); - checkMapping(index, "", "", root, fields); + checkMapping(index, type, "", "", root, fields); AtomicInteger empty = new AtomicInteger(); Map map = sortByValue(fields); map.forEach((key, value) -> { logger.info("{} {} {}", - key, value, (double) value * 100 / total.value); + key, + value, + (double) value * 100 / total); if (value == 0) { empty.incrementAndGet(); } @@ -944,7 +616,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @SuppressWarnings("unchecked") - private void checkMapping(String index, + private void checkMapping(String index, String type, String pathDef, String fieldName, Map map, Map fields) { String path = pathDef; @@ -969,20 +641,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String fieldType = o instanceof String ? o.toString() : null; // do not recurse into our custom field mapper if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) { - checkMapping(index, path, key, child, fields); + checkMapping(index, type, path, key, child, fields); } } else if ("type".equals(key)) { QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(queryBuilder) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices(index) - .source(builder); + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) + .setIndices(index) + .setQuery(queryBuilder) + .setSize(0) + .setTrackTotalHits(true); SearchResponse searchResponse = - client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); + searchRequestBuilder.execute().actionGet(); fields.put(path, searchResponse.getHits().getTotalHits().value); } } @@ -995,58 +665,38 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return result; } - private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { - switch (timeUnit) { - case DAYS: - return TimeValue.timeValueHours(24 * timeValue); - case HOURS: - return TimeValue.timeValueHours(timeValue); - case MINUTES: - return TimeValue.timeValueMinutes(timeValue); - case SECONDS: - return TimeValue.timeValueSeconds(timeValue); - case MILLISECONDS: - return TimeValue.timeValueMillis(timeValue); - case MICROSECONDS: - return TimeValue.timeValueNanos(1000 * timeValue); - case NANOSECONDS: - return TimeValue.timeValueNanos(timeValue); - default: - throw new IllegalArgumentException("unknown time unit: " + timeUnit); - } - } - private static class SuccessIndexShiftResult implements IndexShiftResult { - List movedAliases; + Collection movedAliases; - List newAliases; + Collection newAliases; - SuccessIndexShiftResult(List movedAliases, List newAliases) { + SuccessIndexShiftResult(Collection movedAliases, Collection newAliases) { this.movedAliases = movedAliases; this.newAliases = newAliases; } @Override - public List getMovedAliases() { + public Collection getMovedAliases() { return movedAliases; } @Override - public List getNewAliases() { + public Collection getNewAliases() { return newAliases; } } private static class SuccessPruneResult implements IndexPruneResult { - List candidateIndices; + Collection candidateIndices; - List indicesToDelete; + Collection indicesToDelete; AcknowledgedResponse response; - SuccessPruneResult(List candidateIndices, List indicesToDelete, + SuccessPruneResult(Collection candidateIndices, + Collection indicesToDelete, AcknowledgedResponse response) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; @@ -1059,48 +709,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public List getCandidateIndices() { + public Collection getCandidateIndices() { return candidateIndices; } @Override - public List getDeletedIndices() { - return indicesToDelete; - } - - @Override - public boolean isAcknowledged() { - return response.isAcknowledged(); - } - } - - private static class FailPruneResult implements IndexPruneResult { - - List candidateIndices; - - List indicesToDelete; - - AcknowledgedResponse response; - - FailPruneResult(List candidateIndices, List indicesToDelete, - AcknowledgedResponse response) { - this.candidateIndices = candidateIndices; - this.indicesToDelete = indicesToDelete; - this.response = response; - } - - @Override - public IndexPruneResult.State getState() { - return IndexPruneResult.State.SUCCESS; - } - - @Override - public List getCandidateIndices() { - return candidateIndices; - } - - @Override - public List getDeletedIndices() { + public Collection getDeletedIndices() { return indicesToDelete; } @@ -1112,11 +726,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static class NothingToDoPruneResult implements IndexPruneResult { - List candidateIndices; + Collection candidateIndices; - List indicesToDelete; + Collection indicesToDelete; - NothingToDoPruneResult(List candidateIndices, List indicesToDelete) { + NothingToDoPruneResult(Collection candidateIndices, List indicesToDelete) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; } @@ -1127,12 +741,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public List getCandidateIndices() { + public Collection getCandidateIndices() { return candidateIndices; } @Override - public List getDeletedIndices() { + public Collection getDeletedIndices() { return indicesToDelete; } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java new file mode 100644 index 0000000..891bb8b --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java @@ -0,0 +1,217 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.xbib.elx.api.BasicClient; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractBasicClient implements BasicClient { + + private static final Logger logger = LogManager.getLogger(AbstractBasicClient.class.getName()); + + protected ElasticsearchClient client; + + protected Settings settings; + + private final AtomicBoolean closed; + + public AbstractBasicClient() { + closed = new AtomicBoolean(false); + } + + @Override + public void setClient(ElasticsearchClient client) { + logger.log(Level.INFO, "setting client = " + client); + this.client = client; + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + @Override + public void init(Settings settings) throws IOException { + if (closed.compareAndSet(false, true)) { + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + this.settings = settings; + setClient(createClient(settings)); + } else { + logger.log(Level.WARN, "not initializing"); + } + } + + @Override + public String getClusterName() { + ensureClientIsPresent(); + try { + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); + ClusterStateResponse clusterStateResponse = + getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + return clusterStateResponse.getClusterName().value(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name(); + if (logger.isErrorEnabled()) { + logger.error(message); + } + throw new IllegalStateException(message); + } + } + @Override + public void waitForShards(long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + logger.info("waiting for cluster shard settling"); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() + .waitForNoInitializingShards(true) + .waitForNoRelocatingShards(true) + .timeout(timeout); + ClusterHealthResponse healthResponse = + client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); + if (healthResponse.isTimedOut()) { + String message = "timeout waiting for cluster shards"; + logger.error(message); + throw new IllegalStateException(message); + } + } + + @Override + public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + try { + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout)).actionGet(); + ClusterHealthStatus status = healthResponse.getStatus(); + return status.name(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public long getSearchableDocs(String index) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) + .setIndices(index) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0) + .setTrackTotalHits(true); + return searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value; + } + + @Override + public boolean isIndexExists(String index) { + IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); + indicesExistsRequest.indices(index); + IndicesExistsResponse indicesExistsResponse = + client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + return indicesExistsResponse.isExists(); + } + + + @Override + public void close() throws IOException { + ensureClientIsPresent(); + if (closed.compareAndSet(false, true)) { + closeClient(settings); + } + } + + protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; + + protected abstract void closeClient(Settings settings) throws IOException; + + protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { + ensureClientIsPresent(); + if (index == null) { + throw new IOException("no index name given"); + } + if (key == null) { + throw new IOException("no key given"); + } + if (value == null) { + throw new IOException("no value given"); + } + Settings.Builder updateSettingsBuilder = Settings.builder(); + updateSettingsBuilder.put(key, value.toString()); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) + .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit)); + client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); + } + + protected void ensureClientIsPresent() { + if (client == null) { + throw new IllegalStateException("no client"); + } + } + + protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { + switch (timeUnit) { + case DAYS: + return TimeValue.timeValueHours(24 * timeValue); + case HOURS: + return TimeValue.timeValueHours(timeValue); + case MINUTES: + return TimeValue.timeValueMinutes(timeValue); + case SECONDS: + return TimeValue.timeValueSeconds(timeValue); + case MILLISECONDS: + return TimeValue.timeValueMillis(timeValue); + case MICROSECONDS: + return TimeValue.timeValueNanos(1000 * timeValue); + case NANOSECONDS: + return TimeValue.timeValueNanos(timeValue); + default: + throw new IllegalArgumentException("unknown time unit: " + timeUnit); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java new file mode 100644 index 0000000..29d2940 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java @@ -0,0 +1,246 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.xbib.elx.api.BulkClient; +import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.IndexDefinition; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractBulkClient extends AbstractBasicClient implements BulkClient { + + private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName()); + + private BulkMetric bulkMetric; + + private BulkController bulkController; + + private final AtomicBoolean closed = new AtomicBoolean(true); + + @Override + public void init(Settings settings) throws IOException { + if (closed.compareAndSet(true, false)) { + super.init(settings); + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + bulkMetric = new DefaultBulkMetric(); + bulkMetric.init(settings); + bulkController = new DefaultBulkController(this, bulkMetric); + bulkController.init(settings); + } else { + logger.log(Level.WARN, "not initializing"); + } + } + + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + + @Override + public BulkController getBulkController() { + return bulkController; + } + + @Override + public void flush() throws IOException { + if (bulkController != null) { + bulkController.flush(); + } + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + ensureClientIsPresent(); + if (bulkMetric != null) { + logger.info("closing bulk metric"); + bulkMetric.close(); + } + if (bulkController != null) { + logger.info("closing bulk controller"); + bulkController.close(); + } + closeClient(settings); + } + } + + @Override + public void newIndex(IndexDefinition indexDefinition) throws IOException { + Settings settings = indexDefinition.getSettings() == null ? null : + Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build(); + Map mappings = indexDefinition.getMappings() == null ? null : + JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered(); + newIndex(indexDefinition.getFullIndexName(), settings, mappings); + } + + @Override + public void newIndex(String index) throws IOException { + newIndex(index, Settings.EMPTY, (Map) null); + } + + @Override + public void newIndex(String index, Settings settings) throws IOException { + newIndex(index, settings, (Map) null); + } + + @Override + public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException { + String mappingString = Strings.toString(builder); + Map mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered(); + newIndex(index, settings, mappings); + } + + @Override + public void newIndex(String index, Settings settings, Map mapping) throws IOException { + if (index == null) { + logger.warn("no index name given to create index"); + return; + } + ensureClientIsPresent(); + waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); + CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); + if (settings != null) { + createIndexRequest.settings(settings); + } + if (mapping != null) { + createIndexRequest.mapping("_doc", mapping); + } + CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + if (createIndexResponse.isAcknowledged()) { + logger.info("index {} created", index); + } + } + + @Override + public void startBulk(IndexDefinition indexDefinition) throws IOException { + startBulk(indexDefinition.getFullIndexName(), -1, 1); + } + + @Override + public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) + throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); + } + } + + @Override + public void stopBulk(IndexDefinition indexDefinition) throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.stopBulkMode(indexDefinition); + } + } + + @Override + public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.stopBulkMode(index, timeout, timeUnit); + } + } + + @Override + public BulkClient index(String index, String id, boolean create, String source) { + return index(index, id, create, new BytesArray(source.getBytes(StandardCharsets.UTF_8))); + } + + @Override + public BulkClient index(String index, String id, boolean create, BytesReference source) { + return index(new IndexRequest().index(index).id(id).create(create) + .source(source, XContentType.JSON)); + } + + @Override + public BulkClient index(IndexRequest indexRequest) { + ensureClientIsPresent(); + bulkController.bulkIndex(indexRequest); + return this; + } + + @Override + public BulkClient delete(String index, String id) { + return delete(new DeleteRequest().index(index).id(id)); + } + + @Override + public BulkClient delete(DeleteRequest deleteRequest) { + ensureClientIsPresent(); + bulkController.bulkDelete(deleteRequest); + return this; + } + + @Override + public BulkClient update(String index, String id, BytesReference source) { + return update(new UpdateRequest().index(index).id(id) + .doc(source, XContentType.JSON)); + } + + @Override + public BulkClient update(String index, String id, String source) { + return update(new UpdateRequest().index(index).id(id) + .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); + } + + @Override + public BulkClient update(UpdateRequest updateRequest) { + ensureClientIsPresent(); + bulkController.bulkUpdate(updateRequest); + return this; + } + + @Override + public boolean waitForResponses(long timeout, TimeUnit timeUnit) { + ensureClientIsPresent(); + return bulkController.waitForBulkResponses(timeout, timeUnit); + } + + @Override + public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { + super.updateIndexSetting(index, key, value, timeout, timeUnit); + } + + @Override + public void flushIndex(String index) { + if (index != null) { + ensureClientIsPresent(); + client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + } + + @Override + public void refreshIndex(String index) { + if (index != null) { + ensureClientIsPresent(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java new file mode 100644 index 0000000..4dfd3b7 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java @@ -0,0 +1,158 @@ +package org.xbib.elx.common; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetAction; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.search.ClearScrollAction; +import org.elasticsearch.action.search.ClearScrollRequestBuilder; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import org.xbib.elx.api.SearchClient; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Optional; +import java.util.Spliterator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class AbstractSearchClient extends AbstractBasicClient implements SearchClient { + + @Override + public Optional get(Consumer getRequestBuilderConsumer) { + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE); + getRequestBuilderConsumer.accept(getRequestBuilder); + GetResponse getResponse = getRequestBuilder.execute().actionGet(); + return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty(); + } + + @Override + public Optional multiGet(Consumer multiGetRequestBuilderConsumer) { + MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE); + multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder); + MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet(); + return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse); + } + + @Override + public Optional search(Consumer queryBuilder) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + queryBuilder.accept(searchRequestBuilder); + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + if (searchResponse.getFailedShards() > 0) { + StringBuilder sb = new StringBuilder("Search failed:"); + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + sb.append("\n").append(failure.reason()); + } + throw new ElasticsearchException(sb.toString()); + } + return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse); + } + + @Override + public Stream search(Consumer queryBuilder, + TimeValue scrollTime, int scrollSize) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + queryBuilder.accept(searchRequestBuilder); + searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize); + SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet(); + Stream infiniteResponses = Stream.iterate(originalSearchResponse, + searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE) + .setScrollId(searchResponse.getScrollId()) + .setScroll(scrollTime) + .execute().actionGet()); + Predicate condition = searchResponse -> searchResponse.getHits().getHits().length > 0; + Consumer lastAction = searchResponse -> { + ClearScrollRequestBuilder clearScrollRequestBuilder = + new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE) + .addScrollId(searchResponse.getScrollId()); + clearScrollRequestBuilder.execute().actionGet(); + }; + return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(), + condition, lastAction), false) + .onClose(infiniteResponses::close) + .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits())); + } + + @Override + public Stream getIds(Consumer queryBuilder) { + return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId); + } + + static class TakeWhileSpliterator implements Spliterator { + + private final Spliterator source; + + private final Predicate condition; + + private final Consumer lastElement; + + private final boolean inclusive; + + private final AtomicBoolean checked; + + static TakeWhileSpliterator over(Spliterator source, + Predicate condition, + Consumer lastElement) { + return new TakeWhileSpliterator<>(source, condition, lastElement, true); + } + + private TakeWhileSpliterator(Spliterator source, + Predicate condition, + Consumer lastElement, + boolean inclusive) { + this.source = source; + this.condition = condition; + this.lastElement = lastElement; + this.inclusive = inclusive; + this.checked = new AtomicBoolean(true); + } + + @Override + public boolean tryAdvance(Consumer action) { + return checked.get() && source.tryAdvance(e -> { + if (condition.test(e)) { + action.accept(e); + } else { + if (inclusive && checked.get()) { + action.accept(e); + } + lastElement.accept(e); + checked.set(false); + } + }); + } + + @Override + public Spliterator trySplit() { + return null; + } + + @Override + public long estimateSize() { + return checked.get() ? source.estimateSize() : 0; + } + + @Override + public int characteristics() { + return source.characteristics() &~ Spliterator.SIZED; + } + + @Override + public Comparator getComparator() { + return source.getComparator(); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java index e77a55a..6816713 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -8,11 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elx.api.ExtendedClient; -import org.xbib.elx.api.ExtendedClientProvider; +import org.xbib.elx.api.AdminClientProvider; +import org.xbib.elx.api.BulkClientProvider; +import org.xbib.elx.api.BasicClient; +import org.xbib.elx.api.SearchClientProvider; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; @@ -23,11 +24,15 @@ public class ClientBuilder { private final ElasticsearchClient client; + private final ClassLoader classLoader; + private final Settings.Builder settingsBuilder; - private Map, ExtendedClientProvider> providerMap; + private Class adminClientProvider; - private Class provider; + private Class bulkClientProvider; + + private Class searchClientProvider; public ClientBuilder() { this(null); @@ -39,14 +44,9 @@ public class ClientBuilder { public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) { this.client = client; + this.classLoader = classLoader; this.settingsBuilder = Settings.builder(); settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT); - this.providerMap = new HashMap<>(); - ServiceLoader serviceLoader = ServiceLoader.load(ExtendedClientProvider.class, - classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader()); - for (ExtendedClientProvider provider : serviceLoader) { - providerMap.put(provider.getClass(), provider); - } } public static ClientBuilder builder() { @@ -57,8 +57,18 @@ public class ClientBuilder { return new ClientBuilder(client); } - public ClientBuilder provider(Class provider) { - this.provider = provider; + public ClientBuilder setAdminClientProvider(Class adminClientProvider) { + this.adminClientProvider = adminClientProvider; + return this; + } + + public ClientBuilder setBulkClientProvider(Class bulkClientProvider) { + this.bulkClientProvider = bulkClientProvider; + return this; + } + + public ClientBuilder setSearchClientProvider(Class searchClientProvider) { + this.searchClientProvider = searchClientProvider; return this; } @@ -97,15 +107,59 @@ public class ClientBuilder { return this; } - @SuppressWarnings("unchecked") - public C build() throws IOException { - if (provider == null) { - throw new IllegalArgumentException("no provider"); + public ClientBuilder put(Map map) { + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() == null) { + continue; + } + if (entry.getValue() instanceof String || + entry.getValue() instanceof Integer || + entry.getValue() instanceof Long || + entry.getValue() instanceof Float || + entry.getValue() instanceof TimeValue) { + settingsBuilder.put(entry.getKey(), entry.getValue().toString()); + } else { + logger.log(Level.WARN, "skipping " + entry.getValue() + + " because invalid class type " + entry.getValue().getClass().getName()); + } } + return this; + } + + @SuppressWarnings("unchecked") + public C build() throws IOException { Settings settings = settingsBuilder.build(); logger.log(Level.INFO, "settings = " + settings.toDelimitedString(',')); - return (C) providerMap.get(provider).getExtendedClient() - .setClient(client) - .init(settings); + if (adminClientProvider != null) { + for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(adminClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + if (bulkClientProvider != null) { + for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(bulkClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + if (searchClientProvider != null) { + for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(searchClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + throw new IllegalArgumentException("no provider"); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index 968d678..0420fce 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -8,11 +8,11 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elx.api.BulkClient; import org.xbib.elx.api.BulkController; import org.xbib.elx.api.BulkListener; import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkProcessor; -import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexDefinition; import java.io.IOException; @@ -27,7 +27,7 @@ public class DefaultBulkController implements BulkController { private static final Logger logger = LogManager.getLogger(DefaultBulkController.class); - private final ExtendedClient client; + private final BulkClient bulkClient; private final BulkMetric bulkMetric; @@ -45,9 +45,9 @@ public class DefaultBulkController implements BulkController { private final AtomicBoolean active; - public DefaultBulkController(ExtendedClient client) { - this.client = client; - this.bulkMetric = new DefaultBulkMetric(); + public DefaultBulkController(BulkClient bulkClient, BulkMetric bulkMetric) { + this.bulkClient = bulkClient; + this.bulkMetric = bulkMetric; this.indexNames = new ArrayList<>(); this.active = new AtomicBoolean(false); this.startBulkRefreshIntervals = new HashMap<>(); @@ -80,7 +80,7 @@ public class DefaultBulkController implements BulkController { boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), Parameters.ENABLE_BULK_LOGGING.getValue()); BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging); - this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) + this.bulkProcessor = DefaultBulkProcessor.builder(bulkClient.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) .setConcurrentRequests(maxConcurrentRequests) .setFlushInterval(flushIngestInterval) @@ -115,7 +115,7 @@ public class DefaultBulkController implements BulkController { startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds); stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds); if (startRefreshIntervalInSeconds != 0L) { - client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s", + bulkClient.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s", 30L, TimeUnit.SECONDS); } } @@ -187,7 +187,7 @@ public class DefaultBulkController implements BulkController { if (indexNames.contains(index)) { Long secs = stopBulkRefreshIntervals.get(index); if (secs != null && secs != 0L) { - client.updateIndexSetting(index, "refresh_interval", secs + "s", + bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s", 30L, TimeUnit.SECONDS); } indexNames.remove(index); @@ -206,11 +206,11 @@ public class DefaultBulkController implements BulkController { public void close() throws IOException { flush(); bulkMetric.close(); - if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) { + if (bulkClient.waitForResponses(maxWaitTime, maxWaitTimeUnit)) { for (String index : indexNames) { Long secs = stopBulkRefreshIntervals.get(index); if (secs != null && secs != 0L) - client.updateIndexSetting(index, "refresh_interval", secs + "s", + bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s", 30L, TimeUnit.SECONDS); } indexNames.clear(); diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java index 1350e65..8127e29 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java @@ -1,5 +1,6 @@ package org.xbib.elx.common; +import org.elasticsearch.common.settings.Settings; import org.xbib.elx.api.BulkMetric; import org.xbib.metrics.api.Count; import org.xbib.metrics.api.Metered; @@ -36,6 +37,10 @@ public class DefaultBulkMetric implements BulkMetric { submitted = new CountMetric(); succeeded = new CountMetric(); failed = new CountMetric(); + } + + @Override + public void init(Settings settings) { start(); } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java index 52127e1..8d2d3fd 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java @@ -3,8 +3,6 @@ package org.xbib.elx.common; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexRetention; -import java.net.MalformedURLException; -import java.net.URL; import java.util.concurrent.TimeUnit; public class DefaultIndexDefinition implements IndexDefinition { @@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition { private String dateTimePattern; - private URL settingsUrl; + private String settings; - private URL mappingsUrl; + private String mappings; private boolean enabled; @@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition { } @Override - public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { - this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; + public IndexDefinition setSettings(String settings) { + this.settings = settings; return this; } @Override - public IndexDefinition setSettingsUrl(URL settingsUrl) { - this.settingsUrl = settingsUrl; + public String getSettings() { + return settings; + } + + @Override + public IndexDefinition setMappings(String mappings) { + this.mappings = mappings; return this; } @Override - public URL getSettingsUrl() { - return settingsUrl; - } - - @Override - public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException { - this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null; - return this; - } - - @Override - public IndexDefinition setMappingsUrl(URL mappingsUrl) { - this.mappingsUrl = mappingsUrl; - return this; - } - - @Override - public URL getMappingsUrl() { - return mappingsUrl; + public String getMappings() { + return mappings; } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java new file mode 100644 index 0000000..ebc1450 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java @@ -0,0 +1,59 @@ +package org.xbib.elx.common; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockAdminClient extends AbstractAdminClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient(Settings settings) { + } + + @Override + public MockAdminClient deleteIndex(String index) { + return this; + } + + @Override + public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { + } + + @Override + public void waitForShards(long maxWaitTime, TimeUnit timeUnit) { + + } + + @Override + public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { + return this; + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java new file mode 100644 index 0000000..ca1917b --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.AdminClientProvider; + +public class MockAdminClientProvider implements AdminClientProvider { + @Override + public MockAdminClient getClient() { + return new MockAdminClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java new file mode 100644 index 0000000..5494d90 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java @@ -0,0 +1,103 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockBulkClient extends AbstractBulkClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + public String getClusterName() { + return null; + } + + @Override + public void waitForShards(long maxWaitTime, TimeUnit timeUnit) { + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient(Settings settings) { + } + + @Override + public MockBulkClient index(String index, String id, boolean create, String source) { + return this; + } + + @Override + public MockBulkClient delete(String index, String id) { + return this; + } + + @Override + public MockBulkClient update(String index, String id, String source) { + return this; + } + + @Override + public MockBulkClient index(IndexRequest indexRequest) { + return this; + } + + @Override + public MockBulkClient delete(DeleteRequest deleteRequest) { + return this; + } + + @Override + public MockBulkClient update(UpdateRequest updateRequest) { + return this; + } + + @Override + public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { + } + + @Override + public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { + } + + @Override + public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public void refreshIndex(String index) { + } + + @Override + public void flushIndex(String index) { + } + + + @Override + public void flush() { + // nothing to do + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java new file mode 100644 index 0000000..93e032d --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.BulkClientProvider; + +public class MockBulkClientProvider implements BulkClientProvider { + + @Override + public MockBulkClient getClient() { + return new MockBulkClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java deleted file mode 100644 index 647894b..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.xbib.elx.common; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; - -import java.util.concurrent.TimeUnit; - -/** - * A mocked client, it does not perform any actions on a cluster. Useful for testing. - */ -public class MockExtendedClient extends AbstractExtendedClient { - - @Override - public ElasticsearchClient getClient() { - return null; - } - - @Override - public MockExtendedClient init(Settings settings) { - return this; - } - - @Override - protected ElasticsearchClient createClient(Settings settings) { - return null; - } - - @Override - protected void closeClient() { - } - - @Override - public MockExtendedClient index(String index, String id, boolean create, String source) { - return this; - } - - @Override - public MockExtendedClient delete(String index, String id) { - return this; - } - - @Override - public MockExtendedClient update(String index, String id, String source) { - return this; - } - - @Override - public MockExtendedClient index(IndexRequest indexRequest) { - return this; - } - - @Override - public MockExtendedClient delete(DeleteRequest deleteRequest) { - return this; - } - - @Override - public MockExtendedClient update(UpdateRequest updateRequest) { - return this; - } - - @Override - public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { - return this; - } - - @Override - public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { - return this; - } - - @Override - public MockExtendedClient newIndex(String index) { - return this; - } - - @Override - public MockExtendedClient deleteIndex(String index) { - return this; - } - - @Override - public MockExtendedClient refreshIndex(String index) { - return this; - } - - @Override - public MockExtendedClient flushIndex(String index) { - return this; - } - - @Override - public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { - return this; - } - - @Override - public void flush() { - // nothing to do - } - - @Override - public void close() { - // nothing to do - } -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java deleted file mode 100644 index 87e65cc..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.common; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class MockExtendedClientProvider implements ExtendedClientProvider { - @Override - public MockExtendedClient getExtendedClient() { - return new MockExtendedClient(); - } -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java new file mode 100644 index 0000000..9d182dc --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java @@ -0,0 +1,44 @@ +package org.xbib.elx.common; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockSearchClient extends AbstractSearchClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + public String getClusterName() { + return null; + } + + @Override + public void waitForShards(long maxWaitTime, TimeUnit timeUnit) { + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient(Settings settings) { + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java new file mode 100644 index 0000000..c943fb4 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.SearchClientProvider; + +public class MockSearchClientProvider implements SearchClientProvider { + + @Override + public MockSearchClient getClient() { + return new MockSearchClient(); + } +} diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..1a2c887 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockAdminClientProvider \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..23d86e7 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockBulkClientProvider \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 9729b83..0000000 --- a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.common.MockExtendedClientProvider \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..ec8036b --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockSearchClientProvider \ No newline at end of file diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockClientProviderTest.java new file mode 100644 index 0000000..194770a --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockClientProviderTest.java @@ -0,0 +1,41 @@ +package org.xbib.elx.common.test; + +import org.junit.jupiter.api.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.MockAdminClient; +import org.xbib.elx.common.MockAdminClientProvider; +import org.xbib.elx.common.MockBulkClient; +import org.xbib.elx.common.MockBulkClientProvider; +import org.xbib.elx.common.MockSearchClient; +import org.xbib.elx.common.MockSearchClientProvider; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class MockClientProviderTest { + + @Test + void testMockAdminProvider() throws IOException { + MockAdminClient client = ClientBuilder.builder() + .setAdminClientProvider(MockAdminClientProvider.class) + .build(); + assertNotNull(client); + } + + @Test + void testMockBulkProvider() throws IOException { + MockBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(MockBulkClientProvider.class) + .build(); + assertNotNull(client); + } + + @Test + void testMockSearchProvider() throws IOException { + MockSearchClient client = ClientBuilder.builder() + .setSearchClientProvider(MockSearchClientProvider.class) + .build(); + assertNotNull(client); + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java deleted file mode 100644 index e21817c..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.xbib.elx.common.test; - -import org.junit.jupiter.api.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.MockExtendedClient; -import org.xbib.elx.common.MockExtendedClientProvider; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -class MockExtendedClientProviderTest { - - @Test - void testMockExtendedProvider() throws IOException { - MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); - assertNotNull(client); - } -} diff --git a/elx-common/src/test/resources/log4j2.xml b/elx-common/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-common/src/test/resources/log4j2.xml rename to elx-common/src/test/resources/log4j2-test.xml diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java deleted file mode 100644 index 628ba4f..0000000 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.xbib.elx.http; - -import org.xbib.elx.api.ExtendedClientProvider; - -import java.util.Collections; - -public class ExtendedHttpClientProvider implements ExtendedClientProvider { - @Override - public ExtendedHttpClient getExtendedClient() { - return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader()); - } -} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java index 93e6fa9..5773b0a 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java @@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse; */ public class HttpActionContext { - private final ExtendedHttpClient extendedHttpClient; + private final HttpClientHelper extendedHttpClient; private final R request; @@ -23,13 +23,13 @@ public class HttpActionContext ActionFuture execute(ActionType action, Request request) { + return helper.execute(action, request); + } + + @Override + public void execute(ActionType action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java new file mode 100644 index 0000000..b544ea9 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.AdminClientProvider; + +public class HttpAdminClientProvider implements AdminClientProvider { + + @Override + public HttpAdminClient getClient() { + return new HttpAdminClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java new file mode 100644 index 0000000..a9a9ac0 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java @@ -0,0 +1,55 @@ +package org.xbib.elx.http; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +/** + * Elasticsearch HTTP bulk client. + */ +public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient { + + private final HttpClientHelper helper; + + public HttpBulkClient() { + this.helper = new HttpClientHelper(); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init(settings); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return this; + } + + @Override + protected void closeClient(Settings settings) throws IOException { + helper.closeClient(settings); + } + + @Override + public ActionFuture execute(ActionType action, Request request) { + return helper.execute(action, request); + } + + @Override + public void execute(ActionType action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java new file mode 100644 index 0000000..cbf1945 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.BulkClientProvider; + +public class HttpBulkClientProvider implements BulkClientProvider { + + @Override + public HttpBulkClient getClient() { + return new HttpBulkClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java similarity index 84% rename from elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java rename to elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java index bf9aa8e..35208e0 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java @@ -10,16 +10,15 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; -import org.xbib.elx.common.AbstractExtendedClient; import org.xbib.net.URL; import org.xbib.netty.http.client.Client; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,9 +30,9 @@ import java.util.stream.Stream; /** * Elasticsearch HTTP client. */ -public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient { +public class HttpClientHelper { - private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class); + private static final Logger logger = LogManager.getLogger(HttpClientHelper.class); private Client nettyHttpClient; @@ -46,17 +45,20 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti private String url; - public ExtendedHttpClient(List namedXContentEntries, ClassLoader classLoader) { + public HttpClientHelper() { + this(Collections.emptyList(), Thread.currentThread().getContextClassLoader()); + } + + public HttpClientHelper(List namedXContentEntries, + ClassLoader classLoader) { this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(), namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList())); this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader(); this.actionMap = new HashMap<>(); } - @Override @SuppressWarnings({"unchecked", "rawtypes"}) - public ExtendedHttpClient init(Settings settings) throws IOException { - super.init(settings); + public HttpClientHelper init(Settings settings) throws IOException { if (settings.hasValue("url")) { this.url = settings.get("url"); } else if (settings.hasValue("host")) { @@ -92,22 +94,10 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti return nettyHttpClient; } - @Override - public ElasticsearchClient getClient() { - return this; - } - - @Override - protected ElasticsearchClient createClient(Settings settings) { - return this; - } - - @Override - protected void closeClient() throws IOException { + protected void closeClient(Settings settings) throws IOException { nettyHttpClient.shutdownGracefully(); } - @Override public ActionFuture execute(ActionType action, Request request) { PlainActionFuture actionFuture = PlainActionFuture.newFuture(); @@ -115,13 +105,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti return actionFuture; } - @Override public void execute(ActionType action, Request request, ActionListener listener) { doExecute(action, request, listener); } - @Override public ThreadPool threadPool() { logger.log(Level.DEBUG, "returning null for threadPool() request"); return null; diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java new file mode 100644 index 0000000..0cff238 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java @@ -0,0 +1,55 @@ +package org.xbib.elx.http; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +/** + * Elasticsearch HTTP search client. + */ +public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient { + + private final HttpClientHelper helper; + + public HttpSearchClient() { + this.helper = new HttpClientHelper(); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init(settings); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return this; + } + + @Override + protected void closeClient(Settings settings) throws IOException { + helper.closeClient(settings); + } + + @Override + public ActionFuture execute(ActionType action, Request request) { + return helper.execute(action, request); + } + + @Override + public void execute(ActionType action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java new file mode 100644 index 0000000..9103c1a --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.SearchClientProvider; + +public class HttpSearchClientProvider implements SearchClientProvider { + + @Override + public HttpSearchClient getClient() { + return new HttpSearchClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/get/HtppGetIndexAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/get/HtppGetIndexAction.java new file mode 100644 index 0000000..f97dd0d --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/get/HtppGetIndexAction.java @@ -0,0 +1,35 @@ +package org.xbib.elx.http.action.admin.indices.get; + +import org.elasticsearch.action.admin.indices.get.GetIndexAction; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.xcontent.XContentParser; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class HtppGetIndexAction extends HttpAction { + + @Override + public GetIndexAction getActionInstance() { + return GetIndexAction.INSTANCE; + } + + @Override + protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException { + List list = getIndexRequest.indices().length == 0 ? + List.of("*") : Arrays.asList(getIndexRequest.indices()); + String command = "/" + String.join(",", list); + logger.info("command = " + command); + return newGetRequest(url, command); + } + + @Override + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return GetIndexResponse::fromXContent; + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpClearScrollAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpClearScrollAction.java new file mode 100644 index 0000000..b7d5f16 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpClearScrollAction.java @@ -0,0 +1,35 @@ +package org.xbib.elx.http.action.search; + +import org.elasticsearch.action.search.ClearScrollAction; +import org.elasticsearch.action.search.ClearScrollRequest; +import org.elasticsearch.action.search.ClearScrollResponse; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; +import java.io.IOException; + +public class HttpClearScrollAction extends HttpAction { + + @Override + public ClearScrollAction getActionInstance() { + return ClearScrollAction.INSTANCE; + } + + @Override + protected Request.Builder createHttpRequest(String baseUrl, ClearScrollRequest request) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder(); + request.toXContent(builder, ToXContent.EMPTY_PARAMS); + return newDeleteRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder)); + } + + @Override + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return ClearScrollResponse::fromXContent; + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java index 4398195..da2d3b0 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java @@ -5,6 +5,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.Scroll; import org.xbib.elx.http.HttpAction; import org.xbib.netty.http.client.api.Request; import org.xbib.netty.http.common.HttpResponse; @@ -20,9 +21,10 @@ public class HttpSearchAction extends HttpAction @Override protected Request.Builder createHttpRequest(String url, SearchRequest request) { - // request.indices() always empty array + Scroll scroll = request.scroll(); + String params = scroll != null ? "?scroll=" + scroll.keepAlive() : ""; String index = request.indices() != null ? String.join(",", request.indices()) + "/" : ""; - return newPostRequest(url, index + "_search", request.source().toString()); + return newPostRequest(url, index + "_search" + params, request.source().toString()); } @Override diff --git a/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchScrollAction.java b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchScrollAction.java new file mode 100644 index 0000000..382cba3 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchScrollAction.java @@ -0,0 +1,35 @@ +package org.xbib.elx.http.action.search; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.api.Request; +import org.xbib.netty.http.common.HttpResponse; +import java.io.IOException; + +public class HttpSearchScrollAction extends HttpAction { + + @Override + public SearchScrollAction getActionInstance() { + return SearchScrollAction.INSTANCE; + } + + @Override + protected Request.Builder createHttpRequest(String baseUrl, SearchScrollRequest request) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder(); + request.toXContent(builder, ToXContent.EMPTY_PARAMS); + return newPostRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder)); + } + + @Override + protected CheckedFunction entityParser(HttpResponse httpResponse) { + return SearchResponse::fromXContent; + } +} diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..f09a6e4 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpAdminClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..39fbdef --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpBulkClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 0c75f14..0000000 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.http.ExtendedHttpClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..882ed38 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpSearchClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction index 6f8771f..1a16493 100644 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -8,12 +8,15 @@ org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction +org.xbib.elx.http.action.admin.indices.get.HtppGetIndexAction org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction org.xbib.elx.http.action.bulk.HttpBulkAction org.xbib.elx.http.action.index.HttpIndexAction +org.xbib.elx.http.action.search.HttpClearScrollAction org.xbib.elx.http.action.search.HttpSearchAction +org.xbib.elx.http.action.search.HttpSearchScrollAction org.xbib.elx.http.action.main.HttpMainAction org.xbib.elx.http.action.get.HttpExistsAction org.xbib.elx.http.action.get.HttpGetAction diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java new file mode 100644 index 0000000..1a3c231 --- /dev/null +++ b/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java @@ -0,0 +1,182 @@ +package org.xbib.elx.http.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName()); + + private static final Long ACTIONS = 100L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + client.newIndex("test"); + client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + client.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + client.newIndex("test"); + client.close(); + } + + @Test + void testMapping() throws Exception { + try (HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) + .put(helper.getHttpSettings()) + .build(); + HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .build()) { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties")); + } + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + client.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + client.refreshIndex("test"); + assertEquals(numactions, client.getSearchableDocs("test")); + client.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final long actions = ACTIONS; + logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + logger.info("index created"); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(60L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(60L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(60L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java deleted file mode 100644 index cb0b2cf..0000000 --- a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java +++ /dev/null @@ -1,244 +0,0 @@ -package org.xbib.elx.http.test; - -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.Parameters; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; - -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@ExtendWith(TestExtension.class) -class ClientTest { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); - - private static final Long ACTIONS = 100L; - - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - - private final TestExtension.Helper helper; - - ClientTest(TestExtension.Helper helper) { - this.helper = helper; - } - - @Test - void testSingleDoc() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put(helper.getHttpSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .build(); - try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - void testNewIndex() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put(helper.getHttpSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - client.newIndex("test"); - client.close(); - } - - @Test - void testNewIndexWithSettings() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put(helper.getHttpSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); - client.newIndex("test", settings); - GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices("test"); - GetSettingsResponse getSettingsResponse = - client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); - logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); - assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); - client.close(); - } - - @Test - void testNewIndexWithSettingsAndMappings() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put(helper.getHttpSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); - XContentBuilder builder = JsonXContent.contentBuilder() - .startObject() - .field("date_detection", false) - .startObject("properties") - .startObject("pos") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject(); - client.newIndex("test", settings, builder); - GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices("test"); - GetSettingsResponse getSettingsResponse = - client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); - assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest() - .indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); - client.close(); - } - - @Test - void testRandomDocs() throws Exception { - long numactions = ACTIONS; - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put(helper.getHttpSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test"); - for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value); - client.close(); - } - } - - @Test - void testThreadedRandomDocs() throws Exception { - int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final long actions = ACTIONS; - logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put(helper.getHttpSettings()) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - Settings settings = Settings.builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - client.newIndex("test", settings, (String)null) - .startBulk("test", 0, 1000); - logger.info("index created"); - ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); - final CountDownLatch latch = new CountDownLatch(maxthreads); - for (int i = 0; i < maxthreads; i++) { - executorService.execute(() -> { - for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - latch.countDown(); - }); - } - logger.info("waiting for latch..."); - if (latch.await(60L, TimeUnit.SECONDS)) { - logger.info("flush..."); - client.flush(); - client.waitForResponses(60L, TimeUnit.SECONDS); - logger.info("got all responses, executor service shutdown..."); - executorService.shutdown(); - executorService.awaitTermination(60L, TimeUnit.SECONDS); - logger.info("pool is shut down"); - } else { - logger.warn("latch timeout"); - } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - logger.log(Level.INFO, "refreshing index test"); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices("test") - .source(builder); - SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value); - client.close(); - client.close(); - } - } -} diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java index 2fd356c..44cff60 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java @@ -2,18 +2,12 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -26,10 +20,10 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 100L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; + private final TestExtension.Helper helper; DuplicateIDTest(TestExtension.Helper helper) { @@ -39,8 +33,8 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); @@ -53,23 +47,11 @@ class DuplicateIDTest { client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices("test") - .source(builder); - SearchResponse searchResponse = - helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits().value; - logger.info("hits = {}", hits); + long hits = client.getSearchableDocs("test"); assertTrue(hits < ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { client.close(); - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java index 3068771..207953f 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,50 +35,49 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1")) + final HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) + .put(helper.getHttpSettings()) + .build(); + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) - .provider(ExtendedHttpClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1", settings); - client.shiftIndex("test", "test1", Collections.emptyList()); - client.newIndex("test2", settings); - client.shiftIndex("test", "test2", Collections.emptyList()); - client.newIndex("test3", settings); - client.shiftIndex("test", "test3", Collections.emptyList()); - client.newIndex("test4", settings); - client.shiftIndex("test", "test4", Collections.emptyList()); + bulkClient.newIndex("test1", settings); + adminClient.shiftIndex("test", "test1", Collections.emptyList()); + bulkClient.newIndex("test2", settings); + adminClient.shiftIndex("test", "test2", Collections.emptyList()); + bulkClient.newIndex("test3", settings); + adminClient.shiftIndex("test", "test3", Collections.emptyList()); + bulkClient.newIndex("test4", settings); + adminClient.shiftIndex("test", "test4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test", "test4", 2, 2, true); + adminClient.pruneIndex("test", "test4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); List list = new ArrayList<>(); for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { - IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(index); - IndicesExistsResponse indicesExistsResponse = - client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); - logger.info("indices exists response for {} is {}", index, indicesExistsResponse.isExists()); - list.add(indicesExistsResponse.isExists()); + list.add(bulkClient.isIndexExists(index)); } + logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.close(); } } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java index 234c0de..25ca051 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java @@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -35,56 +35,54 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() + final HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) + .put(helper.getHttpSettings()) + .build(); + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) - .provider(ExtendedHttpClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1234", settings); + bulkClient.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", helper.randomString(1), false, + bulkClient.index("test1234", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); - + adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - - Map aliases = client.getAliases("test1234"); + Map aliases = adminClient.getAliases("test1234"); logger.log(Level.DEBUG, "aliases = " + aliases); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - String resolved = client.resolveAlias("test"); + String resolved = adminClient.resolveAlias("test"); logger.log(Level.DEBUG, "resolved = " + resolved); - aliases = client.getAliases(resolved); + aliases = adminClient.getAliases(resolved); logger.log(Level.DEBUG, "aliases = " + aliases); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - client.newIndex("test5678", settings); + bulkClient.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", helper.randomString(1), false, + bulkClient.index("test5678", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - - indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -94,31 +92,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - - aliases = client.getAliases("test5678"); + aliases = adminClient.getAliases("test5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - resolved = client.resolveAlias("test"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.close(); } } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java new file mode 100644 index 0000000..1b126e9 --- /dev/null +++ b/elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java @@ -0,0 +1,82 @@ +package org.xbib.elx.http.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; +import org.xbib.elx.http.HttpSearchClient; +import org.xbib.elx.http.HttpSearchClientProvider; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +@ExtendWith(TestExtension.class) +class SearchTest { + + private static final Logger logger = LogManager.getLogger(SearchTest.class.getName()); + + private static final Long ACTIONS = 100L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + + private final TestExtension.Helper helper; + + SearchTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testDocStream() throws Exception { + long numactions = ACTIONS; + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try (bulkClient) { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + } + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + try (HttpSearchClient searchClient = ClientBuilder.builder() + .setSearchClientProvider(HttpSearchClientProvider.class) + .put(helper.getHttpSettings()) + .build()) { + Stream stream = searchClient.search(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery()), + TimeValue.timeValueMinutes(1), 10); + long count = stream.count(); + assertEquals(numactions, count); + Stream ids = searchClient.getIds(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery())); + final AtomicInteger idcount = new AtomicInteger(); + ids.forEach(id -> { + logger.info(id); + idcount.incrementAndGet(); + }); + assertEquals(numactions, idcount.get()); + } + } +} diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java index e0d8879..bd2072f 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,47 +30,46 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + try (HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) .put(helper.getHttpSettings()) .build(); - try { - assertEquals(helper.getClusterName(), client.getClusterName()); - client.newIndex("test_smoke"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); - client.delete("test_smoke", "1"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() - .build()); + HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .build()) { + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); assertEquals(0, indexDefinition.getReplicaLevel()); - client.newIndex(indexDefinition); - client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); + assertEquals(helper.getClusterName(), adminClient.getClusterName()); + bulkClient.newIndex("test_smoke"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.deleteIndex("test_smoke"); + bulkClient.newIndex(indexDefinition); + bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.updateReplicaLevel(indexDefinition, 2); + int replica = adminClient.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); - assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.deleteIndex(indexDefinition); } } } diff --git a/elx-http/src/test/resources/log4j2.xml b/elx-http/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-http/src/test/resources/log4j2.xml rename to elx-http/src/test/resources/log4j2-test.xml diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java deleted file mode 100644 index 9d35b9b..0000000 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.xbib.elx.node; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.env.Environment; -import org.elasticsearch.node.Node; -import org.elasticsearch.plugins.Plugin; -import org.xbib.elx.common.AbstractExtendedClient; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -public class ExtendedNodeClient extends AbstractExtendedClient { - - private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName()); - - private Node node; - - @Override - protected ElasticsearchClient createClient(Settings settings) throws IOException { - if (settings == null) { - return null; - } - String version = System.getProperty("os.name") - + " " + System.getProperty("java.vm.name") - + " " + System.getProperty("java.vm.vendor") - + " " + System.getProperty("java.runtime.version") - + " " + System.getProperty("java.vm.version"); - Settings effectiveSettings = Settings.builder().put(settings) - .put("node.client", true) - .put("node.master", false) - .put("node.data", false) - .build(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); - logger.info("creating node client on {} with effective settings {}", - version, Strings.toString(builder)); - Collection> plugins = Collections.emptyList(); - this.node = new BulkNode(new Environment(effectiveSettings, null), plugins); - try { - node.start(); - } catch (Exception e) { - throw new IOException(e); - } - return node.client(); - } - - @Override - protected void closeClient() throws IOException { - if (node != null) { - logger.debug("closing node client"); - node.close(); - } - } - - private static class BulkNode extends Node { - - BulkNode(Environment env, Collection> classpathPlugins) { - super(env, classpathPlugins, false); - } - } -} diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java deleted file mode 100644 index 46a4e9a..0000000 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.node; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedNodeClientProvider implements ExtendedClientProvider { - @Override - public ExtendedNodeClient getExtendedClient() { - return new ExtendedNodeClient(); - } -} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java new file mode 100644 index 0000000..b8614a7 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java @@ -0,0 +1,25 @@ +package org.xbib.elx.node; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractAdminClient; +import java.io.IOException; + +public class NodeAdminClient extends AbstractAdminClient { + + private final NodeClientHelper helper; + + public NodeAdminClient() { + this.helper = new NodeClientHelper(); + } + + @Override + public ElasticsearchClient createClient(Settings settings) { + return helper.createClient(settings, null); + } + + @Override + public void closeClient(Settings settings) throws IOException { + helper.closeClient(settings); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java new file mode 100644 index 0000000..ec949e0 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.AdminClientProvider; + +public class NodeAdminClientProvider implements AdminClientProvider { + + @Override + public NodeAdminClient getClient() { + return new NodeAdminClient(); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java new file mode 100644 index 0000000..d8163fe --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java @@ -0,0 +1,25 @@ +package org.xbib.elx.node; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +public class NodeBulkClient extends AbstractBulkClient { + + private final NodeClientHelper helper; + + public NodeBulkClient() { + this.helper = new NodeClientHelper(); + } + + @Override + public ElasticsearchClient createClient(Settings settings) { + return helper.createClient(settings, null); + } + + @Override + public void closeClient(Settings settings) throws IOException { + helper.closeClient(settings); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java new file mode 100644 index 0000000..f40a021 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.BulkClientProvider; + +public class NodeBulkClientProvider implements BulkClientProvider { + + @Override + public NodeBulkClient getClient() { + return new NodeBulkClient(); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeClientHelper.java b/elx-node/src/main/java/org/xbib/elx/node/NodeClientHelper.java new file mode 100644 index 0000000..f1894f1 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeClientHelper.java @@ -0,0 +1,98 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.Netty4Plugin; +import org.xbib.elx.common.Parameters; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class NodeClientHelper { + + private static final Logger logger = LogManager.getLogger(NodeClientHelper.class.getName()); + + private static Object configurationObject; + + private static Node node; + + private static final Map clientMap = new HashMap<>(); + + public ElasticsearchClient createClient(Settings settings, Object object) { + if (configurationObject == null) { + configurationObject = object; + } + if (configurationObject instanceof ElasticsearchClient) { + return (ElasticsearchClient) configurationObject; + } + return clientMap.computeIfAbsent(settings.get("cluster.name"), + key -> innerCreateClient(settings)); + } + + public void closeClient(Settings settings) throws IOException { + ElasticsearchClient client = clientMap.remove(settings.get("cluster.name")); + if (client != null) { + logger.debug("closing node..."); + node.close(); + node = null; + } + } + + private ElasticsearchClient innerCreateClient(Settings settings) { + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder() + .put(settings.filter(key -> !isPrivateSettings(key))) + .put("node.master", false) + .put("node.data", false) + // "node.processors" + .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), + settings.get(EsExecutors.NODE_PROCESSORS_SETTING.getKey(), + String.valueOf(Runtime.getRuntime().availableProcessors()))) + // "transport.type" + .put(NetworkModule.TRANSPORT_TYPE_KEY, + Netty4Plugin.NETTY_TRANSPORT_NAME) + .build(); + logger.info("creating node client on {} with effective settings {}", + version, effectiveSettings.toDelimitedString(',')); + Collection> plugins = + Collections.singletonList(Netty4Plugin.class); + node = new BulkNode(new Environment(effectiveSettings, null), plugins); + try { + node.start(); + return node.client(); + } catch (NodeValidationException e) { + logger.log(Level.ERROR, e.getMessage(), e); + } + return null; + } + + private static boolean isPrivateSettings(String key) { + return key.equals(Parameters.MAX_ACTIONS_PER_REQUEST.name()) || + key.equals(Parameters.MAX_CONCURRENT_REQUESTS.name()) || + key.equals(Parameters.MAX_VOLUME_PER_REQUEST.name()) || + key.equals(Parameters.FLUSH_INTERVAL.name()); + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, classpathPlugins, false); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java new file mode 100644 index 0000000..ed0954c --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java @@ -0,0 +1,25 @@ +package org.xbib.elx.node; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +public class NodeSearchClient extends AbstractSearchClient { + + private final NodeClientHelper helper; + + public NodeSearchClient() { + this.helper = new NodeClientHelper(); + } + + @Override + public ElasticsearchClient createClient(Settings settings) { + return helper.createClient(settings, null); + } + + @Override + public void closeClient(Settings settings) throws IOException { + helper.closeClient(settings); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java new file mode 100644 index 0000000..233c2a4 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.SearchClientProvider; + +public class NodeSearchClientProvider implements SearchClientProvider { + + @Override + public NodeSearchClient getClient() { + return new NodeSearchClient(); + } +} diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..acc3b0c --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeAdminClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..62630f5 --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeBulkClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 372aaad..0000000 --- a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.node.ExtendedNodeClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..54b437c --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeSearchClientProvider \ No newline at end of file diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java new file mode 100644 index 0000000..92103f5 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java @@ -0,0 +1,180 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName()); + + private static final Long ACTIONS = 10000L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + bulkClient.newIndex("test"); + bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + bulkClient.newIndex("test"); + bulkClient.close(); + } + + @Test + void testMapping() throws Exception { + try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .put(helper.getNodeSettings()) + .build(); + NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .build()) { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties")); + } + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + final long actions = ACTIONS; + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(30L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(30L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java deleted file mode 100644 index 3b301d6..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ /dev/null @@ -1,236 +0,0 @@ -package org.xbib.elx.node.test; - -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.Parameters; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@ExtendWith(TestExtension.class) -class ClientTest { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - - private static final Long ACTIONS = 10000L; - - private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; - - private final TestExtension.Helper helper; - - ClientTest(TestExtension.Helper helper) { - this.helper = helper; - } - - @Test - void testSingleDoc() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .build(); - try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - void testNewIndex() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - client.newIndex("test"); - client.close(); - } - - @Test - void testNewIndexWithSettings() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); - client.newIndex("test", settings); - GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices("test"); - GetSettingsResponse getSettingsResponse = - client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); - logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); - assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); - client.close(); - } - - @Test - void testNewIndexWithSettingsAndMappings() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); - XContentBuilder builder = JsonXContent.contentBuilder() - .startObject() - .field("date_detection", false) - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject(); - client.newIndex("test", settings, builder); - GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices("test"); - GetSettingsResponse getSettingsResponse = - client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); - logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); - assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest() - .indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); - client.close(); - } - - @Test - void testRandomDocs() throws Exception { - long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test"); - for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value); - client.close(); - } - } - - @Test - void testThreadedRandomDocs() throws Exception { - int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final long actions = ACTIONS; - logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - Settings settings = Settings.builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - client.newIndex("test", settings, (String)null) - .startBulk("test", 0, 1000); - logger.info("index created"); - ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); - final CountDownLatch latch = new CountDownLatch(maxthreads); - for (int i = 0; i < maxthreads; i++) { - executorService.execute(() -> { - for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - latch.countDown(); - }); - } - logger.info("waiting for latch..."); - if (latch.await(30L, TimeUnit.SECONDS)) { - logger.info("flush..."); - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - logger.info("got all responses, executor service shutdown..."); - executorService.shutdown(); - executorService.awaitTermination(30L, TimeUnit.SECONDS); - logger.info("pool is shut down"); - } else { - logger.warn("latch timeout"); - } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - logger.log(Level.INFO, "refreshing index test"); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices("test") - .source(builder); - SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value); - client.close(); - } - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index 80c15d0..d8d76f9 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -2,18 +2,12 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -26,10 +20,10 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 100L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; + private final TestExtension.Helper helper; DuplicateIDTest(TestExtension.Helper helper) { @@ -39,40 +33,28 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { - client.newIndex("test"); + bulkClient.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", helper.randomString(1), false, + bulkClient.index("test", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices("test") - .source(builder); - SearchResponse searchResponse = - helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits().value; - logger.info("hits = {}", hits); - assertTrue(hits < ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test"); + assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS); } finally { - client.close(); - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 43e87ea..c4eb863 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,49 +35,49 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .put(helper.getNodeSettings()) + .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_prune1", settings); - client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); - client.newIndex("test_prune2", settings); - client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); - client.newIndex("test_prune3", settings); - client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); - client.newIndex("test_prune4", settings); - client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + bulkClient.newIndex("test_prune1", settings); + adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + bulkClient.newIndex("test_prune2", settings); + adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + bulkClient.newIndex("test_prune3", settings); + adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + bulkClient.newIndex("test_prune4", settings); + adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test_prune", "test_prune4", 2, 2, true); + adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); List list = new ArrayList<>(); for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { - IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(index); - IndicesExistsResponse indicesExistsResponse = - client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); - list.add(indicesExistsResponse.isExists()); + list.add(bulkClient.isIndexExists(index)); } logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index fb7aaaf..6187e90 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -3,15 +3,16 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -33,50 +34,51 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .put(helper.getNodeSettings()) + .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_shift", settings); + bulkClient.newIndex("test_shift", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift", helper.randomString(1), false, + bulkClient.index("test_shift", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); + adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - - Map aliases = client.getAliases("test_shift"); + Map aliases = adminClient.getAliases("test_shift"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - String resolved = client.resolveAlias("test"); - aliases = client.getAliases(resolved); + String resolved = adminClient.resolveAlias("test"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - client.newIndex("test_shift2", settings); + bulkClient.newIndex("test_shift2", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift2", helper.randomString(1), false, + bulkClient.index("test_shift2", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - - indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -86,32 +88,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - - aliases = client.getAliases("test_shift2"); + aliases = adminClient.getAliases("test_shift2"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - resolved = client.resolveAlias("test"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java new file mode 100644 index 0000000..33dd290 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java @@ -0,0 +1,82 @@ +package org.xbib.elx.node.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; +import org.xbib.elx.node.NodeSearchClient; +import org.xbib.elx.node.NodeSearchClientProvider; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +@ExtendWith(TestExtension.class) +class SearchTest { + + private static final Logger logger = LogManager.getLogger(SearchTest.class.getName()); + + private static final Long ACTIONS = 100L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + + private final TestExtension.Helper helper; + + SearchTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testDocStream() throws Exception { + long numactions = ACTIONS; + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try (bulkClient) { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + } + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + try (NodeSearchClient searchClient = ClientBuilder.builder(helper.client("1")) + .setSearchClientProvider(NodeSearchClientProvider.class) + .put(helper.getNodeSettings()) + .build()) { + Stream stream = searchClient.search(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery()), + TimeValue.timeValueMinutes(1), 10); + long count = stream.count(); + assertEquals(numactions, count); + Stream ids = searchClient.getIds(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery())); + final AtomicInteger idcount = new AtomicInteger(); + ids.forEach(id -> { + logger.info(id); + idcount.incrementAndGet(); + }); + assertEquals(numactions, idcount.get()); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index cc698aa..c7badca 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.api.IndexDefinition; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,46 +30,46 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .put(helper.getNodeSettings()) .build(); - try { - assertEquals(helper.getClusterName(), client.getClusterName()); - client.newIndex("test_smoke"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() - .build()); + NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(helper.getNodeSettings()) + .build()) { + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); assertEquals(0, indexDefinition.getReplicaLevel()); - client.newIndex(indexDefinition); - client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); + assertEquals(helper.getClusterName(), adminClient.getClusterName()); + bulkClient.newIndex("test_smoke"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.deleteIndex("test_smoke"); + bulkClient.newIndex(indexDefinition); + bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.updateReplicaLevel(indexDefinition, 2); + int replica = adminClient.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); - assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.deleteIndex(indexDefinition); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java index 1c5b2e5..c8ffcfc 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java @@ -193,6 +193,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft .put("path.home", getHome()) .put("cluster.initial_master_nodes", "1") .put("discovery.seed_hosts", "127.0.0.1:9300") + .put("node.max_local_storage_nodes", "2") .build(); } diff --git a/elx-node/src/test/resources/log4j2.xml b/elx-node/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-node/src/test/resources/log4j2.xml rename to elx-node/src/test/resources/log4j2-test.xml diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java deleted file mode 100644 index 669d21a..0000000 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.xbib.elx.transport; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedTransportClientProvider implements ExtendedClientProvider { - - @Override - public ExtendedTransportClient getExtendedClient() { - return new ExtendedTransportClient(); - } -} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java new file mode 100644 index 0000000..36bdf65 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java @@ -0,0 +1,36 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractAdminClient; + +import java.io.IOException; + +/** + * Transport admin client. + */ +public class TransportAdminClient extends AbstractAdminClient { + + private final TransportClientHelper helper; + + public TransportAdminClient() { + this.helper = new TransportClientHelper(); + } + + @Override + public ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } + + @Override + public void closeClient(Settings settings) { + helper.closeClient(settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java new file mode 100644 index 0000000..18d2368 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.AdminClientProvider; + +public class TransportAdminClientProvider implements AdminClientProvider { + + @Override + public TransportAdminClient getClient() { + return new TransportAdminClient(); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java new file mode 100644 index 0000000..4c48f8d --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java @@ -0,0 +1,35 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +/** + * Transport search client with additional methods. + */ +public class TransportBulkClient extends AbstractBulkClient { + + private final TransportClientHelper helper; + + public TransportBulkClient() { + this.helper = new TransportClientHelper(); + } + + @Override + public ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } + + @Override + public void closeClient(Settings settings) { + helper.closeClient(settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java new file mode 100644 index 0000000..399d73d --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.BulkClientProvider; + +public class TransportBulkClientProvider implements BulkClientProvider { + + @Override + public TransportBulkClient getClient() { + return new TransportBulkClient(); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java similarity index 78% rename from elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java rename to elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java index 28d6d0c..fde33e2 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java @@ -7,6 +7,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.Client; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.client.transport.TransportClient; @@ -23,7 +24,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.transport.Netty4Plugin; -import org.xbib.elx.common.AbstractExtendedClient; import org.xbib.elx.common.util.NetworkUtils; import java.io.IOException; @@ -31,16 +31,19 @@ import java.net.InetAddress; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * Transport client with additional methods using the BulkProcessor. */ -public class ExtendedTransportClient extends AbstractExtendedClient { +public class TransportClientHelper { - private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName()); + private static final Logger logger = LogManager.getLogger(TransportClientHelper.class.getName()); + + private static final Map clientMap = new HashMap<>(); - @Override protected ElasticsearchClient createClient(Settings settings) throws IOException { if (settings != null) { String systemIdentifier = System.getProperty("os.name") @@ -49,11 +52,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient { + " " + System.getProperty("java.vm.version") + " Elasticsearch " + Version.CURRENT.toString(); Settings transportClientSettings = getTransportClientSettings(settings); - //XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject(); XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject(); logger.log(Level.INFO, "creating transport client on {} with settings {}", systemIdentifier, - //Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()), Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder, ToXContent.EMPTY_PARAMS).endObject())); return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class)); @@ -61,29 +62,22 @@ public class ExtendedTransportClient extends AbstractExtendedClient { return null; } - @Override - protected void closeClient() { - if (getClient() != null) { - TransportClient client = (TransportClient) getClient(); - client.close(); - client.threadPool().shutdown(); + public void closeClient(Settings settings) { + ElasticsearchClient client = clientMap.remove(settings.get("cluster.name")); + if (client != null) { + if (client instanceof Client) { + ((Client) client).close(); + } + client.threadPool().shutdownNow(); } } - @Override - public ExtendedTransportClient init(Settings settings) throws IOException { - super.init(settings); - // additional auto-connect - try { - Collection addrs = findAddresses(settings); - if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { - throw new NoNodeAvailableException("no cluster nodes available, check settings " - + settings.toString()); - } - } catch (IOException e) { - logger.error(e.getMessage(), e); + public void init(TransportClient transportClient, Settings settings) throws IOException { + Collection addrs = findAddresses(settings); + if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.toString()); } - return this; } private Collection findAddresses(Settings settings) throws IOException { @@ -112,11 +106,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient { return addresses; } - private boolean connect(Collection addresses, boolean autodiscover) { - if (getClient() == null) { - throw new IllegalStateException("no client present"); - } - TransportClient transportClient = (TransportClient) getClient(); + private boolean connect(TransportClient transportClient, Collection addresses, boolean autodiscover) { for (TransportAddress address : addresses) { transportClient.addTransportAddresses(address); } @@ -126,7 +116,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient { if (autodiscover) { logger.debug("trying to discover all nodes..."); ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); + new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE); ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); for (DiscoveryNode discoveryNode : discoveryNodes) { diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java new file mode 100644 index 0000000..9acf6ef --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java @@ -0,0 +1,35 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +/** + * Transport search client with additional methods. + */ +public class TransportSearchClient extends AbstractSearchClient { + + private final TransportClientHelper helper; + + public TransportSearchClient() { + this.helper = new TransportClientHelper(); + } + + @Override + public ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } + + @Override + public void closeClient(Settings settings) { + helper.closeClient(settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java new file mode 100644 index 0000000..c77e320 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.SearchClientProvider; + +public class TransportSearchClientProvider implements SearchClientProvider { + + @Override + public TransportSearchClient getClient() { + return new TransportSearchClient(); + } +} diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..37ce7ac --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportAdminClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..e1c69bd --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportBulkClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 640e2f9..0000000 --- a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.transport.ExtendedTransportClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..ae6a33b --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportSearchClientProvider \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java new file mode 100644 index 0000000..b3a1df8 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java @@ -0,0 +1,190 @@ +package org.xbib.elx.transport.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName()); + + private static final Long ACTIONS = 10000L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + bulkClient.newIndex("test"); + bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + bulkClient.newIndex("test"); + bulkClient.close(); + } + + @Test + void testMapping() throws Exception { + try (TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build()) { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties")); + } + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + final long actions = ACTIONS; + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .put(Parameters.ENABLE_BULK_LOGGING.name(), "true") + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + logger.info("index created"); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(30L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(30L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java deleted file mode 100644 index 86efd7b..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ /dev/null @@ -1,247 +0,0 @@ -package org.xbib.elx.transport.test; - -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.Parameters; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@ExtendWith(TestExtension.class) -class ClientTest { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - - private static final Long ACTIONS = 10000L; - - private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; - - private final TestExtension.Helper helper; - - ClientTest(TestExtension.Helper helper) { - this.helper = helper; - } - - @Test - void testSingleDoc() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .build(); - try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - void testNewIndex() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - client.newIndex("test"); - client.close(); - } - - @Test - void testNewIndexWithSettings() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); - client.newIndex("test", settings); - GetSettingsRequest getSettingsRequest = new GetSettingsRequest() - .indices("test"); - GetSettingsResponse getSettingsResponse = - client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet(); - logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards")); - assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards")); - client.close(); - } - - @Test - void testNewIndexWithSettingsAndMapping() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - Settings settings = Settings.builder().put("index.number_of_shards", "1").build(); - XContentBuilder builder = JsonXContent.contentBuilder() - .startObject() - .field("date_detection", false) - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject(); - client.newIndex("test", settings, builder); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest() - .indices("test"); - GetMappingsResponse getMappingsResponse = client.getClient() - .execute(GetMappingsAction.INSTANCE, getMappingsRequest) - .actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc")); - client.close(); - } - - @Test - void testRandomDocs() throws Exception { - long numactions = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test"); - for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.source(builder); - SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - logger.log(Level.INFO, searchResponse.toString()); - assertEquals(numactions, searchResponse.getHits().getTotalHits().value); - client.close(); - } - } - - @Test - void testThreadedRandomDocs() throws Exception { - int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final long actions = ACTIONS; - logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .put(Parameters.ENABLE_BULK_LOGGING.name(), "true") - .build(); - try { - Settings settings = Settings.builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - client.newIndex("test", settings, (String)null) - .startBulk("test", 0, 1000); - logger.info("index created"); - ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); - final CountDownLatch latch = new CountDownLatch(maxthreads); - for (int i = 0; i < maxthreads; i++) { - executorService.execute(() -> { - for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - latch.countDown(); - }); - } - logger.info("waiting for latch..."); - if (latch.await(30L, TimeUnit.SECONDS)) { - logger.info("flush..."); - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - logger.info("got all responses, executor service shutdown..."); - executorService.shutdown(); - executorService.awaitTermination(30L, TimeUnit.SECONDS); - logger.info("pool is shut down"); - } else { - logger.warn("latch timeout"); - } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - logger.log(Level.INFO, "refreshing index test"); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices("test") - .source(builder); - SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value); - client.close(); - } - } -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index 9bc45a7..ffd23d2 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -2,18 +2,12 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -26,10 +20,10 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private static final Long ACTIONS = 100L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; + private final TestExtension.Helper helper; DuplicateIDTest(TestExtension.Helper helper) { @@ -39,41 +33,28 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(helper.getTransportSettings()) .build(); try { - client.newIndex("test_dup"); + bulkClient.newIndex("test_dup"); for (int i = 0; i < ACTIONS; i++) { - client.index("test_dup", helper.randomString(1), false, + bulkClient.index("test_dup", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test_dup"); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0) - .trackTotalHits(true); - SearchRequest searchRequest = new SearchRequest() - .indices("test_dup") - .source(builder); - SearchResponse searchResponse = - helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits().value; - logger.info("hits = {}", hits); - assertTrue(hits < ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test_dup"); + assertTrue(bulkClient.getSearchableDocs("test_dup") < ACTIONS); } finally { - client.close(); - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 8d82ba3..0640fbd 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,8 +35,12 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(helper.getTransportSettings()) .build(); try { @@ -46,41 +48,36 @@ class IndexPruneTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_prune1", settings); - client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); - client.newIndex("test_prune2", settings); - client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); - client.newIndex("test_prune3", settings); - client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); - client.newIndex("test_prune4", settings); - client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + bulkClient.newIndex("test_prune1", settings); + adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + bulkClient.newIndex("test_prune2", settings); + adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + bulkClient.newIndex("test_prune3", settings); + adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + bulkClient.newIndex("test_prune4", settings); + adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test_prune", "test_prune4", 2, 2, true); + adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); List list = new ArrayList<>(); for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { - IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(index); - IndicesExistsResponse indicesExistsResponse = - client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); - list.add(indicesExistsResponse.isExists()); + list.add(bulkClient.isIndexExists(index)); } logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index f182956..bf59064 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -3,15 +3,16 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -33,8 +34,12 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(helper.getTransportSettings()) .build(); try { @@ -42,38 +47,38 @@ class IndexShiftTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_shift1234", settings); + bulkClient.newIndex("test_shift1234", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift1234", helper.randomString(1), false, + bulkClient.index("test_shift1234", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c")); + adminClient.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - Map aliases = client.getAliases("test_shift1234"); + Map aliases = adminClient.getAliases("test_shift1234"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test_shift")); - String resolved = client.resolveAlias("test_shift"); - aliases = client.getAliases(resolved); + String resolved = adminClient.resolveAlias("test_shift"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test_shift")); - client.newIndex("test_shift5678", settings); + bulkClient.newIndex("test_shift5678", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift5678", helper.randomString(1), false, + bulkClient.index("test_shift5678", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = client.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"), + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + indexShiftResult = adminClient.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -83,29 +88,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - aliases = client.getAliases("test_shift5678"); + aliases = adminClient.getAliases("test_shift5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - resolved = client.resolveAlias("test_shift"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test_shift"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java new file mode 100644 index 0000000..c923a57 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SearchTest.java @@ -0,0 +1,82 @@ +package org.xbib.elx.transport.test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchHit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; +import org.xbib.elx.transport.TransportSearchClient; +import org.xbib.elx.transport.TransportSearchClientProvider; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +@ExtendWith(TestExtension.class) +class SearchTest { + + private static final Logger logger = LogManager.getLogger(SearchTest.class.getName()); + + private static final Long ACTIONS = 100L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + + private final TestExtension.Helper helper; + + SearchTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testDocStream() throws Exception { + long numactions = ACTIONS; + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .build(); + try (bulkClient) { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + } + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + try (TransportSearchClient searchClient = ClientBuilder.builder() + .setSearchClientProvider(TransportSearchClientProvider.class) + .put(helper.getTransportSettings()) + .build()) { + Stream stream = searchClient.search(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery()), + TimeValue.timeValueMinutes(1), 10); + long count = stream.count(); + assertEquals(numactions, count); + Stream ids = searchClient.getIds(qb -> qb + .setIndices("test") + .setQuery(QueryBuilders.matchAllQuery())); + final AtomicInteger idcount = new AtomicInteger(); + ids.forEach(id -> { + logger.info(id); + idcount.incrementAndGet(); + }); + assertEquals(numactions, idcount.get()); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index 49e5b90..cea2907 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,44 +30,46 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + try (TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) .put(helper.getTransportSettings()) .build(); - try { - assertEquals(helper.getClusterName(), client.getClusterName()); - client.newIndex("test_smoke"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() - .build()); + TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build()) { + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); assertEquals(0, indexDefinition.getReplicaLevel()); - client.newIndex(indexDefinition); - client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); + assertEquals(helper.getClusterName(), adminClient.getClusterName()); + bulkClient.newIndex("test_smoke"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.deleteIndex("test_smoke"); + bulkClient.newIndex(indexDefinition); + bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.updateReplicaLevel(indexDefinition, 2); + int replica = adminClient.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); - assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.deleteIndex(indexDefinition); } } } diff --git a/elx-transport/src/test/resources/log4j2.xml b/elx-transport/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-transport/src/test/resources/log4j2.xml rename to elx-transport/src/test/resources/log4j2-test.xml