From 110c7786da00c710c55137db070f8887a2ca685a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=CC=88rg=20Prante?= Date: Thu, 21 May 2020 17:46:23 +0200 Subject: [PATCH] split into admin, bulk, search client, fix HTTP client --- .../java/org/xbib/elx/api/AdminClient.java | 194 +++++++ ...Provider.java => AdminClientProvider.java} | 2 +- .../java/org/xbib/elx/api/BulkClient.java | 227 ++++++++ .../org/xbib/elx/api/BulkClientProvider.java | 7 + .../java/org/xbib/elx/api/ExtendedClient.java | 480 ---------------- .../xbib/elx/api/ExtendedClientProvider.java | 7 - .../org/xbib/elx/api/IndexDefinition.java | 14 +- .../org/xbib/elx/api/IndexPruneResult.java | 6 +- .../org/xbib/elx/api/IndexShiftResult.java | 6 +- .../java/org/xbib/elx/api/NativeClient.java | 63 +++ .../java/org/xbib/elx/api/SearchClient.java | 28 +- .../xbib/elx/api/SearchClientProvider.java | 7 + ...edClient.java => AbstractAdminClient.java} | 513 +++--------------- .../xbib/elx/common/AbstractBulkClient.java | 246 +++++++++ .../xbib/elx/common/AbstractNativeClient.java | 218 ++++++++ .../xbib/elx/common/AbstractSearchClient.java | 158 ++++++ .../org/xbib/elx/common/ClientBuilder.java | 76 ++- .../elx/common/DefaultBulkController.java | 10 +- .../elx/common/DefaultIndexDefinition.java | 40 +- .../org/xbib/elx/common/MockAdminClient.java | 59 ++ .../elx/common/MockAdminClientProvider.java | 10 + .../org/xbib/elx/common/MockBulkClient.java | 99 ++++ .../elx/common/MockBulkClientProvider.java | 11 + .../xbib/elx/common/MockExtendedClient.java | 129 ----- .../common/MockExtendedClientProvider.java | 10 - .../org/xbib/elx/common/MockSearchClient.java | 38 ++ .../elx/common/MockSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../test/MockAdminClientProviderTest.java | 21 + .../test/MockExtendedClientProviderTest.java | 19 - .../admin/indices/get/HttpGetIndexAction.java | 155 ++++++ .../elx/http/ExtendedHttpClientProvider.java | 12 - .../java/org/xbib/elx/http/HttpAction.java | 6 +- .../org/xbib/elx/http/HttpActionContext.java | 10 +- .../org/xbib/elx/http/HttpAdminClient.java | 61 +++ .../elx/http/HttpAdminClientProvider.java | 10 + .../org/xbib/elx/http/HttpBulkClient.java | 61 +++ .../xbib/elx/http/HttpBulkClientProvider.java | 11 + ...dHttpClient.java => HttpClientHelper.java} | 17 +- .../org/xbib/elx/http/HttpSearchClient.java | 61 +++ .../elx/http/HttpSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../services/org.xbib.elx.http.HttpAction | 1 + .../{ClientTest.java => BulkClientTest.java} | 92 ++-- .../xbib/elx/http/test/DuplicateIDTest.java | 25 +- .../xbib/elx/http/test/IndexPruneTest.java | 53 +- .../xbib/elx/http/test/IndexShiftTest.java | 67 ++- .../org/xbib/elx/http/test/SmokeTest.java | 79 +-- .../resources/{log4j2.xml => log4j2-test.xml} | 0 .../elx/node/ExtendedNodeClientProvider.java | 10 - ...edNodeClient.java => NodeAdminClient.java} | 6 +- .../elx/node/NodeAdminClientProvider.java | 11 + .../org/xbib/elx/node/NodeBulkClient.java | 68 +++ .../xbib/elx/node/NodeBulkClientProvider.java | 11 + .../org/xbib/elx/node/NodeSearchClient.java | 68 +++ .../elx/node/NodeSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../xbib/elx/node/test/BulkClientTest.java | 174 ++++++ .../org/xbib/elx/node/test/ClientTest.java | 200 ------- .../xbib/elx/node/test/DuplicateIDTest.java | 49 +- .../xbib/elx/node/test/IndexPruneTest.java | 52 +- .../xbib/elx/node/test/IndexShiftTest.java | 64 +-- .../org/xbib/elx/node/test/SmokeTest.java | 78 +-- .../resources/{log4j2.xml => log4j2-test.xml} | 0 .../ExtendedTransportClientProvider.java | 11 - .../elx/transport/TransportAdminClient.java | 40 ++ .../TransportAdminClientProvider.java | 11 + .../elx/transport/TransportBulkClient.java | 39 ++ .../TransportBulkClientProvider.java | 11 + ...Client.java => TransportClientHelper.java} | 48 +- .../elx/transport/TransportSearchClient.java | 39 ++ .../TransportSearchClientProvider.java | 11 + .../org.xbib.elx.api.AdminClientProvider | 1 + .../org.xbib.elx.api.BulkClientProvider | 1 + .../org.xbib.elx.api.ExtendedClientProvider | 1 - .../org.xbib.elx.api.SearchClientProvider | 1 + .../elx/transport/test/BulkClientTest.java | 188 +++++++ .../xbib/elx/transport/test/ClientTest.java | 223 -------- .../elx/transport/test/DuplicateIDTest.java | 53 +- .../elx/transport/test/IndexPruneTest.java | 53 +- .../elx/transport/test/IndexShiftTest.java | 58 +- .../xbib/elx/transport/test/SmokeTest.java | 73 +-- 89 files changed, 3003 insertions(+), 2112 deletions(-) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/AdminClient.java rename elx-api/src/main/java/org/xbib/elx/api/{SeachClientProvider.java => AdminClientProvider.java} (52%) create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java delete mode 100644 elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/NativeClient.java create mode 100644 elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java rename elx-common/src/main/java/org/xbib/elx/common/{AbstractExtendedClient.java => AbstractAdminClient.java} (62%) create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java delete mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java delete mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java create mode 100644 elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java create mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider delete mode 100644 elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java delete mode 100644 elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java create mode 100644 elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java delete mode 100644 elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpAdminClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java rename elx-http/src/main/java/org/xbib/elx/http/{ExtendedHttpClient.java => HttpClientHelper.java} (90%) create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java create mode 100644 elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider rename elx-http/src/test/java/org/xbib/elx/http/test/{ClientTest.java => BulkClientTest.java} (60%) rename elx-http/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) delete mode 100644 elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java rename elx-node/src/main/java/org/xbib/elx/node/{ExtendedNodeClient.java => NodeAdminClient.java} (91%) create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java create mode 100644 elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider create mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java delete mode 100644 elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java rename elx-node/src/test/resources/{log4j2.xml => log4j2-test.xml} (100%) delete mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java rename elx-transport/src/main/java/org/xbib/elx/transport/{ExtendedTransportClient.java => TransportClientHelper.java} (77%) create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java create mode 100644 elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider delete mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider create mode 100644 elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider create mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java delete mode 100644 elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java new file mode 100644 index 0000000..bdb5b5d --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java @@ -0,0 +1,194 @@ +package org.xbib.elx.api; + +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Interface for extended managing and indexing methods of an Elasticsearch client. + */ +public interface AdminClient extends NativeClient { + + /** + * Build index definition from settings. + * + * @param index the index name + * @param settings the settings for the index + * @return index definition + * @throws IOException if settings/mapping URL is invalid/malformed + */ + IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; + + /** + * Delete an index. + * @param indexDefinition the index definition + * @return this + */ + AdminClient deleteIndex(IndexDefinition indexDefinition); + + /** + * Delete an index. + * + * @param index index + * @return this + */ + AdminClient deleteIndex(String index); + + + /** + * Update replica level. + * @param indexDefinition the index definition + * @param level the replica level + * @return this + * @throws IOException if replica setting could not be updated + */ + AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; + + /** + * Update replica level. + * + * @param index index + * @param level the replica level + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return this + * @throws IOException if replica setting could not be updated + */ + AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; + + /** + * Get replica level. + * @param indexDefinition the index name + * @return the replica level of the index + */ + int getReplicaLevel(IndexDefinition indexDefinition); + + /** + * Get replica level. + * @param index the index name + * @return the replica level of the index + */ + int getReplicaLevel(String index); + + /** + * Force segment merge of an index. + * @param indexDefinition the index definition + * @return this + */ + boolean forceMerge(IndexDefinition indexDefinition); + + /** + * Force segment merge of an index. + * @param index the index + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return this + */ + boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); + + + /** + * Wait for index recovery (after replica change). + * + * @param index index + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return true if wait succeeded, false if wait timed out + */ + boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit); + + /** + * Resolve alias. + * + * @param alias the alias + * @return this index name behind the alias or the alias if there is no index + */ + String resolveAlias(String alias); + + /** + * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index + * name. + * + * @param alias the alias + * @return the most recent index name pointing to the alias + */ + String resolveMostRecentIndex(String alias); + + /** + * Get all index aliases. + * @param index the index + * @return map of index aliases + */ + Map getAliases(String index); + + /** + * Shift from one index to another. + * @param indexDefinition the index definition + * @param additionalAliases new aliases + * @return this + */ + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); + + /** + * Shift from one index to another. + * @param indexDefinition the index definition + * @param additionalAliases new aliases + * @param indexAliasAdder method to add aliases + * @return this + */ + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, + IndexAliasAdder indexAliasAdder); + + /** + * Shift from one index to another. + * @param index the index name + * @param fullIndexName the index name with timestamp + * @param additionalAliases a list of names that should be set as index aliases + * @return this + */ + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); + + /** + * Shift from one index to another. + * @param index the index name + * @param fullIndexName the index name with timestamp + * @param additionalAliases a list of names that should be set as index aliases + * @param adder an adder method to create alias term queries + * @return this + */ + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, + IndexAliasAdder adder); + + /** + * Prune index. + * @param indexDefinition the index definition + * @return the index prune result + */ + IndexPruneResult pruneIndex(IndexDefinition indexDefinition); + + /** + * Apply retention policy to prune indices. All indices before delta should be deleted, + * but the number of mintokeep indices must be kept. + * + * @param index index name + * @param fullIndexName index name with timestamp + * @param delta timestamp delta (for index timestamps) + * @param mintokeep minimum number of indices to keep + * @param perform true if pruning should be executed, false if not + * @return the index prune result + */ + IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); + + /** + * Find the timestamp of the most recently indexed document in the index. + * + * @param index the index name + * @param timestampfieldname the timestamp field name + * @return millis UTC millis of the most recent document + * @throws IOException if most rcent document can not be found + */ + Long mostRecentDocument(String index, String timestampfieldname) throws IOException; +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/SeachClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java similarity index 52% rename from elx-api/src/main/java/org/xbib/elx/api/SeachClientProvider.java rename to elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java index 08e3f65..954efd5 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/SeachClientProvider.java +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java @@ -1,7 +1,7 @@ package org.xbib.elx.api; @FunctionalInterface -public interface SeachClientProvider { +public interface AdminClientProvider { C getClient(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java new file mode 100644 index 0000000..5f21d14 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java @@ -0,0 +1,227 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.Flushable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public interface BulkClient extends NativeClient, Flushable { + + /** + * Get bulk metric. + * @return the bulk metric + */ + BulkMetric getBulkMetric(); + + /** + * Get buulk control. + * @return the bulk control + */ + BulkController getBulkController(); + + + /** + * Create a new index. + * + * @param index index + * @throws IOException if new index creation fails + */ + void newIndex(String index) throws IOException; + + /** + * Create a new index. + * @param indexDefinition the index definition + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(IndexDefinition indexDefinition) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @throws IOException if settings is invalid or index creation fails + */ + void newIndex(String index, Settings settings) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(String index, Settings settings, Map mapping) throws IOException; + + /** + * Add index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. + * + * @param index the index + * @param id the id + * @param create true if document must be created + * @param source the source + * @return this + */ + BulkClient index(String index, String id, boolean create, BytesReference source); + + /** + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. + * + * @param index the index + * @param id the id + * @param create true if document is to be created, false otherwise + * @param source the source + * @return this client methods + */ + BulkClient index(String index, String id, boolean create, String source); + + /** + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param indexRequest the index request to add + * @return this + */ + BulkClient index(IndexRequest indexRequest); + + /** + * Delete request. + * + * @param index the index + * @param id the id + * @return this + */ + BulkClient delete(String index, String id); + + /** + * Delete request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param deleteRequest the delete request to add + * @return this + */ + BulkClient delete(DeleteRequest deleteRequest); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param index the index + * @param id the id + * @param source the source + * @return this + */ + BulkClient update(String index, String id, BytesReference source); + + /** + * Update document. Use with precaution! Does not work in all cases. + * + * @param index the index + * @param id the id + * @param source the source + * @return this + */ + BulkClient update(String index, String id, String source); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param updateRequest the update request to add + * @return this + */ + BulkClient update(UpdateRequest updateRequest); + + /** + * Start bulk mode for indexes. + * @param indexDefinition index definition + * @throws IOException if bulk could not be started + */ + void startBulk(IndexDefinition indexDefinition) throws IOException; + + /** + * Start bulk mode. + * + * @param index index + * @param startRefreshIntervalSeconds refresh interval before bulk + * @param stopRefreshIntervalSeconds refresh interval after bulk + * @throws IOException if bulk could not be started + */ + void startBulk(String index, long startRefreshIntervalSeconds, + long stopRefreshIntervalSeconds) throws IOException; + + + /** + * Stop bulk mode. + * + * @param indexDefinition index definition + * @throws IOException if bulk could not be startet + */ + void stopBulk(IndexDefinition indexDefinition) throws IOException; + + /** + * Stops bulk mode. + * + * @param index index + * @param timeout maximum wait time + * @param timeUnit time unit for timeout + * @throws IOException if bulk could not be stopped + */ + void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; + + /** + * Wait for all outstanding bulk responses. + * + * @param timeout maximum wait time + * @param timeUnit unit of timeout value + * @return true if wait succeeded, false if wait timed out + */ + boolean waitForResponses(long timeout, TimeUnit timeUnit); + + /** + * Update index setting. + * @param index the index + * @param key the key of the value to be updated + * @param value the new value + * @param timeout timeout + * @param timeUnit time unit + * @throws IOException if update index setting failed + */ + void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; + + /** + * Refresh the index. + * + * @param index index + */ + void refreshIndex(String index); + + /** + * Flush the index. The cluster clears cache and completes indexing. + * + * @param index index + */ + void flushIndex(String index); + +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java new file mode 100644 index 0000000..0f013e2 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface BulkClientProvider { + + C getClient(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java deleted file mode 100644 index a6325f6..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ /dev/null @@ -1,480 +0,0 @@ -package org.xbib.elx.api; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; - -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Interface for extended managing and indexing methods of an Elasticsearch client. - */ -public interface ExtendedClient extends Flushable, Closeable { - - /** - * Set an Elasticsearch client to extend from it. May be null for TransportClient. - * @param client client - * @return this client - */ - ExtendedClient setClient(ElasticsearchClient client); - - /** - * Return Elasticsearch client. - * - * @return Elasticsearch client - */ - ElasticsearchClient getClient(); - - /** - * Get bulk metric. - * @return the bulk metric - */ - BulkMetric getBulkMetric(); - - /** - * Get buulk control. - * @return the bulk control - */ - BulkController getBulkController(); - - /** - * Initiative the extended client, the bulk metric and bulk controller, - * creates instances and connect to cluster, if required. - * - * @param settings settings - * @return this client - * @throws IOException if init fails - */ - ExtendedClient init(Settings settings) throws IOException; - - /** - * Build index definition from settings. - * - * @param index the index name - * @param settings the settings for the index - * @return index definition - * @throws IOException if settings/mapping URL is invalid/malformed - */ - IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; - - /** - * Add index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when limits are exceeded. - * - * @param index the index - * @param id the id - * @param create true if document must be created - * @param source the source - * @return this - */ - ExtendedClient index(String index, String id, boolean create, BytesReference source); - - /** - * Index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when limits are exceeded. - * - * @param index the index - * @param id the id - * @param create true if document is to be created, false otherwise - * @param source the source - * @return this client methods - */ - ExtendedClient index(String index, String id, boolean create, String source); - - /** - * Index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param indexRequest the index request to add - * @return this - */ - ExtendedClient index(IndexRequest indexRequest); - - /** - * Delete request. - * - * @param index the index - * @param id the id - * @return this - */ - ExtendedClient delete(String index, String id); - - /** - * Delete request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param deleteRequest the delete request to add - * @return this - */ - ExtendedClient delete(DeleteRequest deleteRequest); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized. - * - * @param index the index - * @param id the id - * @param source the source - * @return this - */ - ExtendedClient update(String index, String id, BytesReference source); - - /** - * Update document. Use with precaution! Does not work in all cases. - * - * @param index the index - * @param id the id - * @param source the source - * @return this - */ - ExtendedClient update(String index, String id, String source); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized. - * - * @param updateRequest the update request to add - * @return this - */ - ExtendedClient update(UpdateRequest updateRequest); - - /** - * Create a new index. - * - * @param index index - * @return this - * @throws IOException if new index creation fails - */ - ExtendedClient newIndex(String index) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @return this - * @throws IOException if settings is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; - - /** - * Create a new index. - * @param indexDefinition the index definition - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException; - - /** - * Delete an index. - * @param indexDefinition the index definition - * @return this - */ - ExtendedClient deleteIndex(IndexDefinition indexDefinition); - - /** - * Delete an index. - * - * @param index index - * @return this - */ - ExtendedClient deleteIndex(String index); - - /** - * Start bulk mode for indexes. - * @param indexDefinition index definition - * @return this - * @throws IOException if bulk could not be started - */ - ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException; - - /** - * Start bulk mode. - * - * @param index index - * @param startRefreshIntervalSeconds refresh interval before bulk - * @param stopRefreshIntervalSeconds refresh interval after bulk - * @return this - * @throws IOException if bulk could not be started - */ - ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, - long stopRefreshIntervalSeconds) throws IOException; - - /** - * Stop bulk mode. - * - * @param indexDefinition index definition - * @return this - * @throws IOException if bulk could not be startet - */ - ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException; - - /** - * Stops bulk mode. - * - * @param index index - * @param timeout maximum wait time - * @param timeUnit time unit for timeout - * @return this - * @throws IOException if bulk could not be stopped - */ - ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; - - /** - * Update replica level. - * @param indexDefinition the index definition - * @param level the replica level - * @return this - * @throws IOException if replica setting could not be updated - */ - ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; - - /** - * Update replica level. - * - * @param index index - * @param level the replica level - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return this - * @throws IOException if replica setting could not be updated - */ - ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; - - /** - * Get replica level. - * @param indexDefinition the index name - * @return the replica level of the index - */ - int getReplicaLevel(IndexDefinition indexDefinition); - - /** - * Get replica level. - * @param index the index name - * @return the replica level of the index - */ - int getReplicaLevel(String index); - - /** - * Refresh the index. - * - * @param index index - * @return this - */ - ExtendedClient refreshIndex(String index); - - /** - * Flush the index. The cluster clears cache and completes indexing. - * - * @param index index - * @return this - */ - ExtendedClient flushIndex(String index); - - /** - * Force segment merge of an index. - * @param indexDefinition the index definition - * @return this - */ - boolean forceMerge(IndexDefinition indexDefinition); - - /** - * Force segment merge of an index. - * @param index the index - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return this - */ - boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); - - /** - * Wait for all outstanding bulk responses. - * - * @param timeout maximum wait time - * @param timeUnit unit of timeout value - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForResponses(long timeout, TimeUnit timeUnit); - - /** - * Wait for cluster being healthy. - * - * @param healthColor cluster health color to wait for - * @param maxWaitTime time value - * @param timeUnit time unit - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); - - /** - * Get current health color. - * - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return the cluster health color - */ - String getHealthColor(long maxWaitTime, TimeUnit timeUnit); - - /** - * Wait for index recovery (after replica change). - * - * @param index index - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit); - - /** - * Update index setting. - * @param index the index - * @param key the key of the value to be updated - * @param value the new value - * @param timeout timeout - * @param timeUnit time unit - * @throws IOException if update index setting failed - */ - void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; - - /** - * Resolve alias. - * - * @param alias the alias - * @return this index name behind the alias or the alias if there is no index - */ - String resolveAlias(String alias); - - /** - * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index - * name. - * - * @param alias the alias - * @return the most recent index name pointing to the alias - */ - String resolveMostRecentIndex(String alias); - - /** - * Get all index aliases. - * @param index the index - * @return map of index aliases - */ - Map getAliases(String index); - - /** - * Shift from one index to another. - * @param indexDefinition the index definition - * @param additionalAliases new aliases - * @return this - */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); - - /** - * Shift from one index to another. - * @param indexDefinition the index definition - * @param additionalAliases new aliases - * @param indexAliasAdder method to add aliases - * @return this - */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, - IndexAliasAdder indexAliasAdder); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @param adder an adder method to create alias term queries - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, - IndexAliasAdder adder); - - /** - * Prune index. - * @param indexDefinition the index definition - * @return the index prune result - */ - IndexPruneResult pruneIndex(IndexDefinition indexDefinition); - - /** - * Apply retention policy to prune indices. All indices before delta should be deleted, - * but the number of mintokeep indices must be kept. - * - * @param index index name - * @param fullIndexName index name with timestamp - * @param delta timestamp delta (for index timestamps) - * @param mintokeep minimum number of indices to keep - * @param perform true if pruning should be executed, false if not - * @return the index prune result - */ - IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); - - /** - * Find the timestamp of the most recently indexed document in the index. - * - * @param index the index name - * @param timestampfieldname the timestamp field name - * @return millis UTC millis of the most recent document - * @throws IOException if most rcent document can not be found - */ - Long mostRecentDocument(String index, String timestampfieldname) throws IOException; - - /** - * Get cluster name. - * @return the cluster name - */ - String getClusterName(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java deleted file mode 100644 index 2a8904a..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.xbib.elx.api; - -@FunctionalInterface -public interface ExtendedClientProvider { - - C getExtendedClient(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 49544a7..1c77ccb 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -1,7 +1,5 @@ package org.xbib.elx.api; -import java.net.MalformedURLException; -import java.net.URL; import java.util.concurrent.TimeUnit; public interface IndexDefinition { @@ -14,17 +12,13 @@ public interface IndexDefinition { String getFullIndexName(); - IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException; + IndexDefinition setSettings(String settings); - IndexDefinition setSettingsUrl(URL settingsUrl); + String getSettings(); - URL getSettingsUrl(); + IndexDefinition setMappings(String mappings); - IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException; - - IndexDefinition setMappingsUrl(URL mappingsUrl); - - URL getMappingsUrl(); + String getMappings(); IndexDefinition setDateTimePattern(String timeWindow); diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java index 0c118f8..cef89b8 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java @@ -1,6 +1,6 @@ package org.xbib.elx.api; -import java.util.List; +import java.util.Collection; public interface IndexPruneResult { @@ -8,9 +8,9 @@ public interface IndexPruneResult { State getState(); - List getCandidateIndices(); + Collection getCandidateIndices(); - List getDeletedIndices(); + Collection getDeletedIndices(); boolean isAcknowledged(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java index 02a2e8c..6ef58d7 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java @@ -1,10 +1,10 @@ package org.xbib.elx.api; -import java.util.List; +import java.util.Collection; public interface IndexShiftResult { - List getMovedAliases(); + Collection getMovedAliases(); - List getNewAliases(); + Collection getNewAliases(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/NativeClient.java b/elx-api/src/main/java/org/xbib/elx/api/NativeClient.java new file mode 100644 index 0000000..52fbbf3 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/NativeClient.java @@ -0,0 +1,63 @@ +package org.xbib.elx.api; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public interface NativeClient extends Closeable { + + /** + * Set an Elasticsearch client to extend from it. May be null for TransportClient. + * @param client the Elasticsearch client + */ + void setClient(ElasticsearchClient client); + + /** + * Return Elasticsearch client. + * + * @return Elasticsearch client + */ + ElasticsearchClient getClient(); + + /** + * Initiative the extended client, the bulk metric and bulk controller, + * creates instances and connect to cluster, if required. + * + * @param settings settings + * @throws IOException if init fails + */ + void init(Settings settings) throws IOException; + + /** + * Get cluster name. + * @return the cluster name + */ + String getClusterName(); + + /** + * Get current health color. + * + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return the cluster health color + */ + String getHealthColor(long maxWaitTime, TimeUnit timeUnit); + + /** + * Wait for cluster being healthy. + * + * @param healthColor cluster health color to wait for + * @param maxWaitTime time value + * @param timeUnit time unit + */ + void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); + + Map getMapping(String index, String mapping); + + long getSearchableDocs(String index); + + boolean isIndexExists(String index); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java b/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java index 0f018d4..a4b595a 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchClient.java @@ -1,25 +1,27 @@ package org.xbib.elx.api; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Stream; -public interface SearchClient { +public interface SearchClient extends NativeClient { - ActionFuture get(GetRequest getRequest); + Optional get(Consumer getRequestBuilder); - void get(GetRequest request, ActionListener listener); + Optional multiGet(Consumer multiGetRequestBuilder); - ActionFuture multiGet(MultiGetRequest request); + Optional search(Consumer searchRequestBuilder); - void multiGet(MultiGetRequest request, ActionListener listener); + Stream search(Consumer searchRequestBuilder, + TimeValue scrollTime, int scrollSize); - ActionFuture search(SearchRequest request); - - void search(SearchRequest request, ActionListener listener); + Stream getIds(Consumer queryBuilder); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java new file mode 100644 index 0000000..23bfb07 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface SearchClientProvider { + + C getClient(); +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java similarity index 62% rename from elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java rename to elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index 417bc1e..a8b15bc 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -2,10 +2,8 @@ package org.xbib.elx.common; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -18,14 +16,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.flush.FlushAction; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexAction; @@ -34,27 +27,16 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -62,9 +44,8 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -72,9 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; -import org.xbib.elx.api.BulkController; -import org.xbib.elx.api.BulkMetric; -import org.xbib.elx.api.ExtendedClient; +import org.xbib.elx.api.AdminClient; import org.xbib.elx.api.IndexAliasAdder; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexPruneResult; @@ -83,14 +62,16 @@ import org.xbib.elx.api.IndexShiftResult; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -102,33 +83,15 @@ import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -public abstract class AbstractExtendedClient implements ExtendedClient { +public abstract class AbstractAdminClient extends AbstractNativeClient implements AdminClient { - private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); - - /** - * The one and only index type name used in the extended client. - * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". - */ - private static final String TYPE_NAME = "doc"; - - /** - * The Elasticsearch client. - */ - private ElasticsearchClient client; - - private BulkMetric bulkMetric; - - private BulkController bulkController; - - private AtomicBoolean closed; + private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName()); private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { @Override @@ -164,184 +127,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } }; - protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; - - protected abstract void closeClient() throws IOException; - - protected AbstractExtendedClient() { - closed = new AtomicBoolean(false); - } - @Override - public AbstractExtendedClient setClient(ElasticsearchClient client) { - this.client = client; - return this; - } - - @Override - public ElasticsearchClient getClient() { - return client; - } - - @Override - public BulkMetric getBulkMetric() { - return bulkMetric; - } - - @Override - public BulkController getBulkController() { - return bulkController; - } - - @Override - public AbstractExtendedClient init(Settings settings) throws IOException { - logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); - if (client == null) { - client = createClient(settings); - } - if (bulkMetric == null) { - bulkMetric = new DefaultBulkMetric(); - bulkMetric.init(settings); - } - if (bulkController == null) { - bulkController = new DefaultBulkController(this, bulkMetric); - bulkController.init(settings); - } - return this; - } - - @Override - public void flush() throws IOException { - if (bulkController != null) { - bulkController.flush(); - } - } - - @Override - public void close() throws IOException { - ensureActive(); - if (closed.compareAndSet(false, true)) { - if (bulkMetric != null) { - logger.info("closing bulk metric"); - bulkMetric.close(); - } - if (bulkController != null) { - logger.info("closing bulk controller"); - bulkController.close(); - } - closeClient(); - } - } - - @Override - public String getClusterName() { - ensureActive(); - try { - ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); - ClusterStateResponse clusterStateResponse = - client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); - return clusterStateResponse.getClusterName().value(); - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - @Override - public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException { - ensureActive(); - waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); - URL indexSettings = indexDefinition.getSettingsUrl(); - if (indexSettings == null) { - logger.warn("warning while creating index '{}', no settings/mappings", - indexDefinition.getFullIndexName()); - newIndex(indexDefinition.getFullIndexName()); - return this; - } - URL indexMappings = indexDefinition.getMappingsUrl(); - if (indexMappings == null) { - logger.warn("warning while creating index '{}', no mappings", - indexDefinition.getFullIndexName()); - newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null); - return this; - } - try (InputStream indexSettingsInput = indexSettings.openStream(); - InputStream indexMappingsInput = indexMappings.openStream()) { - newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput); - } catch (IOException e) { - if (indexDefinition.ignoreErrors()) { - logger.warn(e.getMessage(), e); - logger.warn("warning while creating index '{}' with settings at {} and mappings at {}", - indexDefinition.getFullIndexName(), indexSettings, indexMappings); - } else { - logger.error("error while creating index '{}' with settings at {} and mappings at {}", - indexDefinition.getFullIndexName(), indexSettings, indexMappings); - throw new IOException(e); - } - } - return this; - } - - @Override - public ExtendedClient newIndex(String index) throws IOException { - return newIndex(index, Settings.EMPTY, (Map) null); - } - - @Override - public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException { - return newIndex(index, - Settings.builder().loadFromStream(".json", settings, true).build(), - mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings) throws IOException { - return newIndex(index, settings, (Map) null); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException { - return newIndex(index, settings, - mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, - DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null); - } - - @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { - ensureActive(); - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); - if (settings != null) { - createIndexRequest.settings(settings); - } - if (mapping != null) { - createIndexRequest.mapping(TYPE_NAME, mapping); - } - CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - logger.info("index {} created: {}", index, - Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS))); - return this; - } - - @Override - public ExtendedClient deleteIndex(IndexDefinition indexDefinition) { + public AdminClient deleteIndex(IndexDefinition indexDefinition) { return deleteIndex(indexDefinition.getFullIndexName()); } @Override - public ExtendedClient deleteIndex(String index) { - ensureActive(); + public AdminClient deleteIndex(String index) { + ensureClientIsPresent(); if (index == null) { logger.warn("no index name given to delete index"); return this; @@ -351,99 +144,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return this; } - @Override - public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException { - startBulk(indexDefinition.getFullIndexName(), -1, 1); - return this; - } - - @Override - public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) - throws IOException { - if (bulkController != null) { - ensureActive(); - bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); - } - return this; - } - - @Override - public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException { - if (bulkController != null) { - ensureActive(); - bulkController.stopBulkMode(indexDefinition); - } - return this; - } - - @Override - public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { - if (bulkController != null) { - ensureActive(); - bulkController.stopBulkMode(index, timeout, timeUnit); - } - return this; - } - - @Override - public ExtendedClient index(String index, String id, boolean create, String source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) - .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); - } - - @Override - public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest(index, TYPE_NAME, id).create(create) - .source(source, XContentType.JSON)); - } - - @Override - public ExtendedClient index(IndexRequest indexRequest) { - ensureActive(); - bulkController.index(indexRequest); - return this; - } - - @Override - public ExtendedClient delete(String index, String id) { - return delete(new DeleteRequest(index, TYPE_NAME, id)); - } - - @Override - public ExtendedClient delete(DeleteRequest deleteRequest) { - ensureActive(); - bulkController.delete(deleteRequest); - return this; - } - - @Override - public ExtendedClient update(String index, String id, BytesReference source) { - return update(new UpdateRequest(index, TYPE_NAME, id) - .doc(source, XContentType.JSON)); - } - - @Override - public ExtendedClient update(String index, String id, String source) { - return update(new UpdateRequest(index, TYPE_NAME, id) - .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); - } - - @Override - public ExtendedClient update(UpdateRequest updateRequest) { - ensureActive(); - bulkController.update(updateRequest); - return this; - } - - @Override - public boolean waitForResponses(long timeout, TimeUnit timeUnit) { - ensureActive(); - return bulkController.waitForResponses(timeout, timeUnit); - } - @Override public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - ensureActive(); + ensureClientIsPresent(); ensureIndexGiven(index); GetSettingsRequest settingsRequest = new GetSettingsRequest(); settingsRequest.indices(index); @@ -466,50 +169,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { - ensureActive(); - ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); - TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); - if (healthResponse != null && healthResponse.isTimedOut()) { - if (logger.isErrorEnabled()) { - logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); - } - return false; - } - return true; - } - - @Override - public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { - ensureActive(); - try { - TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); - ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, - new ClusterHealthRequest().timeout(timeout)).actionGet(); - ClusterHealthStatus status = healthResponse.getStatus(); - return status.name(); - } catch (ElasticsearchTimeoutException e) { - logger.warn(e.getMessage(), e); - return "TIMEOUT"; - } catch (NoNodeAvailableException e) { - logger.warn(e.getMessage(), e); - return "DISCONNECTED"; - } catch (Exception e) { - logger.warn(e.getMessage(), e); - return "[" + e.getMessage() + "]"; - } - } - - @Override - public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException { + public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException { return updateReplicaLevel(indexDefinition.getFullIndexName(), level, indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit()); } @Override - public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { + public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations if (level > 0) { updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); @@ -537,27 +203,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return replica; } - @Override - public ExtendedClient flushIndex(String index) { - if (index != null) { - ensureActive(); - client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); - } - return this; - } - - @Override - public ExtendedClient refreshIndex(String index) { - if (index != null) { - ensureActive(); - client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); - } - return this; - } @Override public String resolveMostRecentIndex(String alias) { - ensureActive(); + ensureClientIsPresent(); if (alias == null) { return null; } @@ -585,7 +234,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String resolveAlias(String alias) { - ensureActive(); + ensureClientIsPresent(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); clusterStateRequest.blocks(false); clusterStateRequest.metaData(true); @@ -627,7 +276,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, IndexAliasAdder adder) { - ensureActive(); + ensureClientIsPresent(); if (index == null) { return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to } @@ -715,12 +364,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (index.equals(fullIndexName)) { return EMPTY_INDEX_PRUNE_RESULT; } - ensureActive(); - GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); + ensureClientIsPresent(); + GetIndexRequestBuilder getIndexRequestBuilder = + new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - logger.info("{} indices", getIndexResponse.getIndices().length); - List candidateIndices = new ArrayList<>(); + logger.info("found {} indices", getIndexResponse.getIndices().length); + Set candidateIndices = new TreeSet<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); if (m.matches() && index.equals(m.group(1)) && !s.equals(fullIndexName)) { @@ -760,7 +410,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public Long mostRecentDocument(String index, String timestampfieldname) { - ensureActive(); + ensureClientIsPresent(); SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.sort(sort); @@ -813,7 +463,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException { - boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); + boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockAdminClient)); String indexName = settings.get("name", index); String fullIndexName; String dateTimePattern = settings.get("dateTimePattern"); @@ -833,8 +483,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .setEnabled(isEnabled) .setIndex(indexName) .setFullIndexName(fullIndexName) - .setSettingsUrl(settings.get("settings")) - .setMappingsUrl(settings.get("mapping")) + .setSettings(findSettingsFrom(settings.get("settings"))) + .setMappings(findMappingsFrom(settings.get("mapping"))) .setDateTimePattern(dateTimePattern) .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) .setShift(settings.getAsBoolean("shift", true)) @@ -845,31 +495,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .setStopRefreshInterval(settings.getAsLong("bulk.stoprefreshinterval", -1L)); } - @Override - public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { - ensureActive(); - if (index == null) { - throw new IOException("no index name given"); + private static String findSettingsFrom(String string) throws IOException { + if (string == null) { + return null; } - if (key == null) { - throw new IOException("no key given"); + try { + URL url = new URL(string); + try (InputStream inputStream = url.openStream()) { + Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build(); + XContentBuilder builder = JsonXContent.contentBuilder(); + settings.toXContent(builder, ToXContent.EMPTY_PARAMS); + return Strings.toString(builder); + } + } catch (MalformedURLException e) { + return string; } - if (value == null) { - throw new IOException("no value given"); - } - Settings.Builder updateSettingsBuilder = Settings.builder(); - updateSettingsBuilder.put(key, value.toString()); - UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) - .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit)); - client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); } - private void ensureActive() { - if (this instanceof MockExtendedClient) { - return; + private static String findMappingsFrom(String string) throws IOException { + if (string == null) { + return null; } - if (client == null) { - throw new IllegalStateException("no client"); + try { + URL url = new URL(string); + try (InputStream inputStream = url.openStream()) { + if (string.endsWith(".json")) { + Map mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered(); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject().map(mappings).endObject(); + return Strings.toString(builder); + } + if (string.endsWith(".yml") || string.endsWith(".yaml")) { + Map mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered(); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject().map(mappings).endObject(); + return Strings.toString(builder); + } + } + return string; + } catch (MalformedInputException e) { + return string; } } @@ -896,7 +563,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } public void checkMapping(String index) { - ensureActive(); + ensureClientIsPresent(); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index); GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); ImmutableOpenMap> map = getMappingsResponse.getMappings(); @@ -992,63 +659,43 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static > Map sortByValue(Map map) { Map result = new LinkedHashMap<>(); - map.entrySet().stream().sorted(Comparator.comparing(Map.Entry::getValue)) + map.entrySet().stream().sorted(Map.Entry.comparingByValue()) .forEachOrdered(e -> result.put(e.getKey(), e.getValue())); return result; } - private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { - switch (timeUnit) { - case DAYS: - return TimeValue.timeValueHours(24 * timeValue); - case HOURS: - return TimeValue.timeValueHours(timeValue); - case MINUTES: - return TimeValue.timeValueMinutes(timeValue); - case SECONDS: - return TimeValue.timeValueSeconds(timeValue); - case MILLISECONDS: - return TimeValue.timeValueMillis(timeValue); - case MICROSECONDS: - return TimeValue.timeValueNanos(1000 * timeValue); - case NANOSECONDS: - return TimeValue.timeValueNanos(timeValue); - default: - throw new IllegalArgumentException("unknown time unit: " + timeUnit); - } - } - private static class SuccessIndexShiftResult implements IndexShiftResult { - List movedAliases; + Collection movedAliases; - List newAliases; + Collection newAliases; - SuccessIndexShiftResult(List movedAliases, List newAliases) { + SuccessIndexShiftResult(Collection movedAliases, Collection newAliases) { this.movedAliases = movedAliases; this.newAliases = newAliases; } @Override - public List getMovedAliases() { + public Collection getMovedAliases() { return movedAliases; } @Override - public List getNewAliases() { + public Collection getNewAliases() { return newAliases; } } private static class SuccessPruneResult implements IndexPruneResult { - List candidateIndices; + Collection candidateIndices; - List indicesToDelete; + Collection indicesToDelete; DeleteIndexResponse response; - SuccessPruneResult(List candidateIndices, List indicesToDelete, + SuccessPruneResult(Collection candidateIndices, + Collection indicesToDelete, DeleteIndexResponse response) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; @@ -1061,12 +708,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public List getCandidateIndices() { + public Collection getCandidateIndices() { return candidateIndices; } @Override - public List getDeletedIndices() { + public Collection getDeletedIndices() { return indicesToDelete; } @@ -1078,11 +725,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static class NothingToDoPruneResult implements IndexPruneResult { - List candidateIndices; + Collection candidateIndices; - List indicesToDelete; + Collection indicesToDelete; - NothingToDoPruneResult(List candidateIndices, List indicesToDelete) { + NothingToDoPruneResult(Collection candidateIndices, List indicesToDelete) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; } @@ -1093,12 +740,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public List getCandidateIndices() { + public Collection getCandidateIndices() { return candidateIndices; } @Override - public List getDeletedIndices() { + public Collection getDeletedIndices() { return indicesToDelete; } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java new file mode 100644 index 0000000..e4c603d --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java @@ -0,0 +1,246 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.xbib.elx.api.BulkClient; +import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.IndexDefinition; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractBulkClient extends AbstractNativeClient implements BulkClient { + + private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName()); + + private BulkMetric bulkMetric; + + private BulkController bulkController; + + @Override + public void init(Settings settings) throws IOException { + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + super.init(settings); + if (bulkMetric == null) { + bulkMetric = new DefaultBulkMetric(); + bulkMetric.init(settings); + } + if (bulkController == null) { + bulkController = new DefaultBulkController(this, bulkMetric); + bulkController.init(settings); + } + } + + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + + @Override + public BulkController getBulkController() { + return bulkController; + } + + @Override + public void flush() throws IOException { + if (bulkController != null) { + bulkController.flush(); + } + } + + @Override + public void close() throws IOException { + ensureClientIsPresent(); + if (closed.compareAndSet(false, true)) { + if (bulkMetric != null) { + logger.info("closing bulk metric"); + bulkMetric.close(); + } + if (bulkController != null) { + logger.info("closing bulk controller"); + bulkController.close(); + } + closeClient(); + } + } + + @Override + public void newIndex(IndexDefinition indexDefinition) throws IOException { + Settings settings = indexDefinition.getSettings() == null ? null : + Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build(); + Map mappings = indexDefinition.getMappings() == null ? null : + JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered(); + newIndex(indexDefinition.getFullIndexName(), settings, mappings); + } + + @Override + public void newIndex(String index) throws IOException { + newIndex(index, Settings.EMPTY, (Map) null); + } + + @Override + public void newIndex(String index, Settings settings) throws IOException { + newIndex(index, settings, (Map) null); + } + + @Override + public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException { + String mappingString = Strings.toString(builder); + Map mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered(); + newIndex(index, settings, mappings); + } + + @Override + public void newIndex(String index, Settings settings, Map mapping) throws IOException { + if (index == null) { + logger.warn("no index name given to create index"); + return; + } + ensureClientIsPresent(); + waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); + CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); + if (settings != null) { + createIndexRequest.settings(settings); + } + if (mapping != null) { + createIndexRequest.mapping(TYPE_NAME, mapping); + } + CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + logger.info("index {} created: {}", index, + Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + @Override + public void startBulk(IndexDefinition indexDefinition) throws IOException { + startBulk(indexDefinition.getFullIndexName(), -1, 1); + } + + @Override + public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) + throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); + } + } + + @Override + public void stopBulk(IndexDefinition indexDefinition) throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.stopBulkMode(indexDefinition); + } + } + + @Override + public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.stopBulkMode(index, timeout, timeUnit); + } + } + + @Override + public BulkClient index(String index, String id, boolean create, String source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); + } + + @Override + public BulkClient index(String index, String id, boolean create, BytesReference source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source, XContentType.JSON)); + } + + @Override + public BulkClient index(IndexRequest indexRequest) { + ensureClientIsPresent(); + bulkController.index(indexRequest); + return this; + } + + @Override + public BulkClient delete(String index, String id) { + return delete(new DeleteRequest(index, TYPE_NAME, id)); + } + + @Override + public BulkClient delete(DeleteRequest deleteRequest) { + ensureClientIsPresent(); + bulkController.delete(deleteRequest); + return this; + } + + @Override + public BulkClient update(String index, String id, BytesReference source) { + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source, XContentType.JSON)); + } + + @Override + public BulkClient update(String index, String id, String source) { + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); + } + + @Override + public BulkClient update(UpdateRequest updateRequest) { + ensureClientIsPresent(); + bulkController.update(updateRequest); + return this; + } + + @Override + public boolean waitForResponses(long timeout, TimeUnit timeUnit) { + ensureClientIsPresent(); + return bulkController.waitForResponses(timeout, timeUnit); + } + + @Override + public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { + super.updateIndexSetting(index, key, value, timeout, timeUnit); + } + + @Override + public void flushIndex(String index) { + if (index != null) { + ensureClientIsPresent(); + client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + } + + @Override + public void refreshIndex(String index) { + if (index != null) { + ensureClientIsPresent(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java new file mode 100644 index 0000000..5c8c599 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java @@ -0,0 +1,218 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.xbib.elx.api.NativeClient; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractNativeClient implements NativeClient { + + private static final Logger logger = LogManager.getLogger(AbstractNativeClient.class.getName()); + + /** + * The one and only index type name used in the extended client. + * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". + */ + protected static final String TYPE_NAME = "doc"; + + protected ElasticsearchClient client; + + protected final AtomicBoolean closed; + + protected AbstractNativeClient() { + closed = new AtomicBoolean(false); + } + + @Override + public void setClient(ElasticsearchClient client) { + this.client = client; + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; + + protected abstract void closeClient() throws IOException; + + + @Override + public void init(Settings settings) throws IOException { + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + if (client == null) { + client = createClient(settings); + } + } + + @Override + public String getClusterName() { + ensureClientIsPresent(); + try { + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); + ClusterStateResponse clusterStateResponse = + getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + return clusterStateResponse.getClusterName().value(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name(); + if (logger.isErrorEnabled()) { + logger.error(message); + } + throw new IllegalStateException(message); + } + } + + @Override + public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + try { + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout)).actionGet(); + ClusterHealthStatus status = healthResponse.getStatus(); + return status.name(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public Map getMapping(String index, String mapping) { + GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) + .setIndices(index) + .setTypes(mapping); + GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet(); + logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap()); + return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap(); + } + + @Override + public long getSearchableDocs(String index) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) + .setIndices(index) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0); + return searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + } + + @Override + public boolean isIndexExists(String index) { + IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); + indicesExistsRequest.indices(index); + IndicesExistsResponse indicesExistsResponse = + client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + return indicesExistsResponse.isExists(); + } + + + @Override + public void close() throws IOException { + ensureClientIsPresent(); + if (closed.compareAndSet(false, true)) { + closeClient(); + } + } + + protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { + ensureClientIsPresent(); + if (index == null) { + throw new IOException("no index name given"); + } + if (key == null) { + throw new IOException("no key given"); + } + if (value == null) { + throw new IOException("no value given"); + } + Settings.Builder updateSettingsBuilder = Settings.builder(); + updateSettingsBuilder.put(key, value.toString()); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) + .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit)); + client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); + } + + protected void ensureClientIsPresent() { + if (this instanceof MockAdminClient) { + return; + } + if (client == null) { + throw new IllegalStateException("no client"); + } + } + + protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { + switch (timeUnit) { + case DAYS: + return TimeValue.timeValueHours(24 * timeValue); + case HOURS: + return TimeValue.timeValueHours(timeValue); + case MINUTES: + return TimeValue.timeValueMinutes(timeValue); + case SECONDS: + return TimeValue.timeValueSeconds(timeValue); + case MILLISECONDS: + return TimeValue.timeValueMillis(timeValue); + case MICROSECONDS: + return TimeValue.timeValueNanos(1000 * timeValue); + case NANOSECONDS: + return TimeValue.timeValueNanos(timeValue); + default: + throw new IllegalArgumentException("unknown time unit: " + timeUnit); + } + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java new file mode 100644 index 0000000..fc37ec9 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java @@ -0,0 +1,158 @@ +package org.xbib.elx.common; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetAction; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.search.ClearScrollAction; +import org.elasticsearch.action.search.ClearScrollRequestBuilder; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import org.xbib.elx.api.SearchClient; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Optional; +import java.util.Spliterator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class AbstractSearchClient extends AbstractNativeClient implements SearchClient { + + @Override + public Optional get(Consumer getRequestBuilderConsumer) { + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE); + getRequestBuilderConsumer.accept(getRequestBuilder); + GetResponse getResponse = getRequestBuilder.execute().actionGet(); + return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty(); + } + + @Override + public Optional multiGet(Consumer multiGetRequestBuilderConsumer) { + MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE); + multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder); + MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet(); + return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse); + } + + @Override + public Optional search(Consumer queryBuilder) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + queryBuilder.accept(searchRequestBuilder); + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + if (searchResponse.getFailedShards() > 0) { + StringBuilder sb = new StringBuilder("Search failed:"); + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + sb.append("\n").append(failure.reason()); + } + throw new ElasticsearchException(sb.toString()); + } + return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse); + } + + @Override + public Stream search(Consumer queryBuilder, + TimeValue scrollTime, int scrollSize) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + queryBuilder.accept(searchRequestBuilder); + searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize); + SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet(); + Stream infiniteResponses = Stream.iterate(originalSearchResponse, + searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE) + .setScrollId(searchResponse.getScrollId()) + .setScroll(scrollTime) + .execute().actionGet()); + Predicate condition = searchResponse -> searchResponse.getHits().getHits().length > 0; + Consumer lastAction = searchResponse -> { + ClearScrollRequestBuilder clearScrollRequestBuilder = + new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE) + .addScrollId(searchResponse.getScrollId()); + clearScrollRequestBuilder.execute().actionGet(); + }; + return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(), + condition, lastAction), false) + .onClose(infiniteResponses::close) + .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits())); + } + + @Override + public Stream getIds(Consumer queryBuilder) { + return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId); + } + + static class TakeWhileSpliterator implements Spliterator { + + private final Spliterator source; + + private final Predicate condition; + + private final Consumer lastElement; + + private final boolean inclusive; + + private final AtomicBoolean checked; + + static TakeWhileSpliterator over(Spliterator source, + Predicate condition, + Consumer lastElement) { + return new TakeWhileSpliterator<>(source, condition, lastElement, true); + } + + private TakeWhileSpliterator(Spliterator source, + Predicate condition, + Consumer lastElement, + boolean inclusive) { + this.source = source; + this.condition = condition; + this.lastElement = lastElement; + this.inclusive = inclusive; + this.checked = new AtomicBoolean(true); + } + + @Override + public boolean tryAdvance(Consumer action) { + return checked.get() && source.tryAdvance(e -> { + if (condition.test(e)) { + action.accept(e); + } else { + if (inclusive && checked.get()) { + action.accept(e); + } + lastElement.accept(e); + checked.set(false); + } + }); + } + + @Override + public Spliterator trySplit() { + return null; + } + + @Override + public long estimateSize() { + return checked.get() ? source.estimateSize() : 0; + } + + @Override + public int characteristics() { + return source.characteristics() &~ Spliterator.SIZED; + } + + @Override + public Comparator getComparator() { + return source.getComparator(); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java index e77a55a..5502e9e 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -8,12 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elx.api.ExtendedClient; -import org.xbib.elx.api.ExtendedClientProvider; +import org.xbib.elx.api.AdminClientProvider; +import org.xbib.elx.api.BulkClientProvider; +import org.xbib.elx.api.NativeClient; +import org.xbib.elx.api.SearchClientProvider; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.ServiceLoader; @SuppressWarnings("rawtypes") @@ -23,11 +23,15 @@ public class ClientBuilder { private final ElasticsearchClient client; + private final ClassLoader classLoader; + private final Settings.Builder settingsBuilder; - private Map, ExtendedClientProvider> providerMap; + private Class adminClientProvider; - private Class provider; + private Class bulkClientProvider; + + private Class searchClientProvider; public ClientBuilder() { this(null); @@ -39,14 +43,9 @@ public class ClientBuilder { public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) { this.client = client; + this.classLoader = classLoader; this.settingsBuilder = Settings.builder(); settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT); - this.providerMap = new HashMap<>(); - ServiceLoader serviceLoader = ServiceLoader.load(ExtendedClientProvider.class, - classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader()); - for (ExtendedClientProvider provider : serviceLoader) { - providerMap.put(provider.getClass(), provider); - } } public static ClientBuilder builder() { @@ -57,8 +56,18 @@ public class ClientBuilder { return new ClientBuilder(client); } - public ClientBuilder provider(Class provider) { - this.provider = provider; + public ClientBuilder setAdminClientProvider(Class adminClientProvider) { + this.adminClientProvider = adminClientProvider; + return this; + } + + public ClientBuilder setBulkClientProvider(Class bulkClientProvider) { + this.bulkClientProvider = bulkClientProvider; + return this; + } + + public ClientBuilder setSearchClientProvider(Class searchClientProvider) { + this.searchClientProvider = searchClientProvider; return this; } @@ -98,14 +107,39 @@ public class ClientBuilder { } @SuppressWarnings("unchecked") - public C build() throws IOException { - if (provider == null) { - throw new IllegalArgumentException("no provider"); - } + public C build() throws IOException { Settings settings = settingsBuilder.build(); logger.log(Level.INFO, "settings = " + settings.toDelimitedString(',')); - return (C) providerMap.get(provider).getExtendedClient() - .setClient(client) - .init(settings); + if (adminClientProvider != null) { + for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(adminClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + if (bulkClientProvider != null) { + for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(bulkClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + if (searchClientProvider != null) { + for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(searchClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + throw new IllegalArgumentException("no provider"); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index 432cae4..557befc 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -11,10 +11,10 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elx.api.BulkClient; import org.xbib.elx.api.BulkController; import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkProcessor; -import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexDefinition; import java.io.IOException; @@ -29,7 +29,7 @@ public class DefaultBulkController implements BulkController { private static final Logger logger = LogManager.getLogger(DefaultBulkController.class); - private final ExtendedClient client; + private final BulkClient client; private final BulkMetric bulkMetric; @@ -39,9 +39,9 @@ public class DefaultBulkController implements BulkController { private final Map stopBulkRefreshIntervals; - private long maxWaitTime; + private final long maxWaitTime; - private TimeUnit maxWaitTimeUnit; + private final TimeUnit maxWaitTimeUnit; private BulkProcessor bulkProcessor; @@ -51,7 +51,7 @@ public class DefaultBulkController implements BulkController { private boolean enableBulkLogging; - public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) { + public DefaultBulkController(BulkClient client, BulkMetric bulkMetric) { this.client = client; this.bulkMetric = bulkMetric; this.indexNames = new ArrayList<>(); diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java index 52127e1..8d2d3fd 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java @@ -3,8 +3,6 @@ package org.xbib.elx.common; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexRetention; -import java.net.MalformedURLException; -import java.net.URL; import java.util.concurrent.TimeUnit; public class DefaultIndexDefinition implements IndexDefinition { @@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition { private String dateTimePattern; - private URL settingsUrl; + private String settings; - private URL mappingsUrl; + private String mappings; private boolean enabled; @@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition { } @Override - public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { - this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; + public IndexDefinition setSettings(String settings) { + this.settings = settings; return this; } @Override - public IndexDefinition setSettingsUrl(URL settingsUrl) { - this.settingsUrl = settingsUrl; + public String getSettings() { + return settings; + } + + @Override + public IndexDefinition setMappings(String mappings) { + this.mappings = mappings; return this; } @Override - public URL getSettingsUrl() { - return settingsUrl; - } - - @Override - public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException { - this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null; - return this; - } - - @Override - public IndexDefinition setMappingsUrl(URL mappingsUrl) { - this.mappingsUrl = mappingsUrl; - return this; - } - - @Override - public URL getMappingsUrl() { - return mappingsUrl; + public String getMappings() { + return mappings; } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java new file mode 100644 index 0000000..a63c989 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java @@ -0,0 +1,59 @@ +package org.xbib.elx.common; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockAdminClient extends AbstractAdminClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient() { + } + + @Override + public MockAdminClient deleteIndex(String index) { + return this; + } + + @Override + public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { + } + + @Override + public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { + return this; + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java new file mode 100644 index 0000000..ca1917b --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.AdminClientProvider; + +public class MockAdminClientProvider implements AdminClientProvider { + @Override + public MockAdminClient getClient() { + return new MockAdminClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java new file mode 100644 index 0000000..7c78f83 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java @@ -0,0 +1,99 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockBulkClient extends AbstractBulkClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + public String getClusterName() { + return null; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient() { + } + + @Override + public MockBulkClient index(String index, String id, boolean create, String source) { + return this; + } + + @Override + public MockBulkClient delete(String index, String id) { + return this; + } + + @Override + public MockBulkClient update(String index, String id, String source) { + return this; + } + + @Override + public MockBulkClient index(IndexRequest indexRequest) { + return this; + } + + @Override + public MockBulkClient delete(DeleteRequest deleteRequest) { + return this; + } + + @Override + public MockBulkClient update(UpdateRequest updateRequest) { + return this; + } + + @Override + public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { + } + + @Override + public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { + } + + @Override + public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public void refreshIndex(String index) { + } + + @Override + public void flushIndex(String index) { + } + + + @Override + public void flush() { + // nothing to do + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java new file mode 100644 index 0000000..93e032d --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.BulkClientProvider; + +public class MockBulkClientProvider implements BulkClientProvider { + + @Override + public MockBulkClient getClient() { + return new MockBulkClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java deleted file mode 100644 index 647894b..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.xbib.elx.common; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; - -import java.util.concurrent.TimeUnit; - -/** - * A mocked client, it does not perform any actions on a cluster. Useful for testing. - */ -public class MockExtendedClient extends AbstractExtendedClient { - - @Override - public ElasticsearchClient getClient() { - return null; - } - - @Override - public MockExtendedClient init(Settings settings) { - return this; - } - - @Override - protected ElasticsearchClient createClient(Settings settings) { - return null; - } - - @Override - protected void closeClient() { - } - - @Override - public MockExtendedClient index(String index, String id, boolean create, String source) { - return this; - } - - @Override - public MockExtendedClient delete(String index, String id) { - return this; - } - - @Override - public MockExtendedClient update(String index, String id, String source) { - return this; - } - - @Override - public MockExtendedClient index(IndexRequest indexRequest) { - return this; - } - - @Override - public MockExtendedClient delete(DeleteRequest deleteRequest) { - return this; - } - - @Override - public MockExtendedClient update(UpdateRequest updateRequest) { - return this; - } - - @Override - public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { - return this; - } - - @Override - public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { - return this; - } - - @Override - public MockExtendedClient newIndex(String index) { - return this; - } - - @Override - public MockExtendedClient deleteIndex(String index) { - return this; - } - - @Override - public MockExtendedClient refreshIndex(String index) { - return this; - } - - @Override - public MockExtendedClient flushIndex(String index) { - return this; - } - - @Override - public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { - return this; - } - - @Override - public void flush() { - // nothing to do - } - - @Override - public void close() { - // nothing to do - } -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java deleted file mode 100644 index 87e65cc..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.common; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class MockExtendedClientProvider implements ExtendedClientProvider { - @Override - public MockExtendedClient getExtendedClient() { - return new MockExtendedClient(); - } -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java new file mode 100644 index 0000000..07c0091 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java @@ -0,0 +1,38 @@ +package org.xbib.elx.common; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockSearchClient extends AbstractSearchClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + public String getClusterName() { + return null; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient() { + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java new file mode 100644 index 0000000..c943fb4 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.SearchClientProvider; + +public class MockSearchClientProvider implements SearchClientProvider { + + @Override + public MockSearchClient getClient() { + return new MockSearchClient(); + } +} diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..1a2c887 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockAdminClientProvider \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 9729b83..0000000 --- a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.common.MockExtendedClientProvider \ No newline at end of file diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java new file mode 100644 index 0000000..c7123e8 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java @@ -0,0 +1,21 @@ +package org.xbib.elx.common.test; + +import org.junit.jupiter.api.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.MockAdminClient; +import org.xbib.elx.common.MockAdminClientProvider; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class MockAdminClientProviderTest { + + @Test + void testMockAdminProvider() throws IOException { + MockAdminClient client = ClientBuilder.builder() + .setAdminClientProvider(MockAdminClientProvider.class) + .build(); + assertNotNull(client); + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java deleted file mode 100644 index e21817c..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.xbib.elx.common.test; - -import org.junit.jupiter.api.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.MockExtendedClient; -import org.xbib.elx.common.MockExtendedClientProvider; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -class MockExtendedClientProviderTest { - - @Test - void testMockExtendedProvider() throws IOException { - MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); - assertNotNull(client); - } -} diff --git a/elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java b/elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java new file mode 100644 index 0000000..3c06622 --- /dev/null +++ b/elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java @@ -0,0 +1,155 @@ +package org.elasticsearch.action.admin.indices.get; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.api.Request; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class HttpGetIndexAction extends HttpAction { + + @Override + public GetIndexAction getActionInstance() { + return GetIndexAction.INSTANCE; + } + + @Override + protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException { + List list = getIndexRequest.indices().length == 0 ? + List.of("*") : Arrays.asList(getIndexRequest.indices()); + String command = "/" + String.join(",", list); + logger.info("command = " + command); + return newGetRequest(url, command); + } + + @Override + protected CheckedFunction entityParser() { + return this::fromXContent; + } + + @Override + protected GetIndexResponse emptyResponse() { + return new GetIndexResponse(); + } + + private GetIndexResponse fromXContent(XContentParser parser) throws IOException { + ImmutableOpenMap.Builder> aliases = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder> mappings = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder settings = ImmutableOpenMap.builder(); + List indices = new ArrayList<>(); + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + while (!parser.isClosed()) { + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + String indexName = parser.currentName(); + indices.add(indexName); + IndexEntry indexEntry = parseIndexEntry(parser); + CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias)); + aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases)); + mappings.put(indexName, indexEntry.indexMappings); + settings.put(indexName, indexEntry.indexSettings); + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } else { + parser.nextToken(); + } + } + return new GetIndexResponse(indices.toArray(new String[0]), + mappings.build(), + aliases.build(), + settings.build()); + } + + private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException { + List indexAliases = null; + ImmutableOpenMap indexMappings = null; + Settings indexSettings = null; + Settings indexDefaultSettings = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + switch (parser.currentName()) { + case "aliases": + indexAliases = parseAliases(parser); + break; + case "mappings": + indexMappings = parseMappings(parser); + break; + case "settings": + indexSettings = Settings.fromXContent(parser); + break; + case "defaults": + indexDefaultSettings = Settings.fromXContent(parser); + break; + default: + parser.skipChildren(); + } + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + } + return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings); + } + + private static List parseAliases(XContentParser parser) throws IOException { + List indexAliases = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + indexAliases.add(AliasMetaData.Builder.fromXContent(parser)); + } + return indexAliases; + } + + private static ImmutableOpenMap parseMappings(XContentParser parser) throws IOException { + ImmutableOpenMap.Builder indexMappings = ImmutableOpenMap.builder(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + String mappingType = parser.currentName(); + indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map())); + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + } + return indexMappings.build(); + } + + private static class IndexEntry { + List indexAliases = new ArrayList<>(); + ImmutableOpenMap indexMappings = ImmutableOpenMap.of(); + Settings indexSettings = Settings.EMPTY; + Settings indexDefaultSettings = Settings.EMPTY; + + IndexEntry(List indexAliases, ImmutableOpenMap indexMappings, + Settings indexSettings, Settings indexDefaultSettings) { + if (indexAliases != null) { + this.indexAliases = indexAliases; + } + if (indexMappings != null) { + this.indexMappings = indexMappings; + } + if (indexSettings != null) { + this.indexSettings = indexSettings; + } + if (indexDefaultSettings != null) { + this.indexDefaultSettings = indexDefaultSettings; + } + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java deleted file mode 100644 index 628ba4f..0000000 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java +++ /dev/null @@ -1,12 +0,0 @@ -package org.xbib.elx.http; - -import org.xbib.elx.api.ExtendedClientProvider; - -import java.util.Collections; - -public class ExtendedHttpClientProvider implements ExtendedClientProvider { - @Override - public ExtendedHttpClient getExtendedClient() { - return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader()); - } -} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java index 816f2bd..e177f2a 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java @@ -77,7 +77,7 @@ public abstract class HttpAction httpActionContext) { try (XContentParser parser = XContentFactory.xContent(XContentType.JSON) - .createParser(httpActionContext.getExtendedHttpClient().getRegistry(), + .createParser(httpActionContext.getHelper().getRegistry(), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) { return errorParser().apply(parser); diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java index 93e6fa9..a032a0a 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java @@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse; */ public class HttpActionContext { - private final ExtendedHttpClient extendedHttpClient; + private final HttpClientHelper helper; private final R request; @@ -23,14 +23,14 @@ public class HttpActionContext> ActionFuture execute(Action action, Request request) { + return helper.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public > RequestBuilder prepareExecute(Action action) { + return helper.prepareExecute(action); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java new file mode 100644 index 0000000..4b606aa --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.AdminClientProvider; + +public class HttpAdminClientProvider implements AdminClientProvider { + @Override + public HttpAdminClient getClient() { + return new HttpAdminClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java new file mode 100644 index 0000000..369853b --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java @@ -0,0 +1,61 @@ +package org.xbib.elx.http; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +/** + * Elasticsearch HTTP bulk client. + */ +public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient { + + private final HttpClientHelper helper; + + public HttpBulkClient() { + this.helper = new HttpClientHelper(); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init(settings); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() throws IOException { + helper.closeClient(); + } + + @Override + public > ActionFuture execute(Action action, Request request) { + return helper.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public > RequestBuilder prepareExecute(Action action) { + return helper.prepareExecute(action); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java new file mode 100644 index 0000000..cbf1945 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.BulkClientProvider; + +public class HttpBulkClientProvider implements BulkClientProvider { + + @Override + public HttpBulkClient getClient() { + return new HttpBulkClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java similarity index 90% rename from elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java rename to elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java index cda7a87..0c76157 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java @@ -15,12 +15,12 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; -import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.elx.common.AbstractAdminClient; import org.xbib.net.URL; import org.xbib.netty.http.client.Client; - import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -32,9 +32,9 @@ import java.util.stream.Stream; /** * Elasticsearch HTTP client. */ -public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient { +public class HttpClientHelper extends AbstractAdminClient implements ElasticsearchClient { - private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class); + private static final Logger logger = LogManager.getLogger(HttpClientHelper.class); private Client nettyHttpClient; @@ -47,7 +47,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti private String url; - public ExtendedHttpClient(List namedXContentEntries, ClassLoader classLoader) { + public HttpClientHelper() { + this(Collections.emptyList(), Thread.currentThread().getContextClassLoader()); + } + + public HttpClientHelper(List namedXContentEntries, ClassLoader classLoader) { this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(), namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList())); this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader(); @@ -56,7 +60,7 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti @Override @SuppressWarnings({"unchecked", "rawtypes"}) - public ExtendedHttpClient init(Settings settings) throws IOException { + public void init(Settings settings) throws IOException { super.init(settings); if (settings.hasValue("url")) { this.url = settings.get("url"); @@ -78,7 +82,6 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti this.nettyHttpClient = clientBuilder.build(); logger.log(Level.DEBUG, "extended HTTP client initialized, settings = {}, url = {}, {} actions", settings, url, actionMap.size()); - return this; } private static List getNamedXContents() { diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java new file mode 100644 index 0000000..07e9249 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java @@ -0,0 +1,61 @@ +package org.xbib.elx.http; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +/** + * Elasticsearch HTTP search client. + */ +public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient { + + private final HttpClientHelper helper; + + public HttpSearchClient() { + this.helper = new HttpClientHelper(); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init(settings); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() throws IOException { + helper.closeClient(); + } + + @Override + public > ActionFuture execute(Action action, Request request) { + return helper.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public > RequestBuilder prepareExecute(Action action) { + return helper.prepareExecute(action); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java new file mode 100644 index 0000000..9103c1a --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.SearchClientProvider; + +public class HttpSearchClientProvider implements SearchClientProvider { + + @Override + public HttpSearchClient getClient() { + return new HttpSearchClient(); + } +} diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..f09a6e4 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpAdminClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..39fbdef --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpBulkClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 0c75f14..0000000 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.http.ExtendedHttpClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..882ed38 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpSearchClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction index eb3a014..3e9602c 100644 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -1,4 +1,5 @@ org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction +org.elasticsearch.action.admin.indices.get.HttpGetIndexAction org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java similarity index 60% rename from elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java rename to elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java index 57a388b..c5e7b66 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java @@ -2,24 +2,16 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -31,9 +23,9 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(TestExtension.class) -class ClientTest { +class BulkClientTest { - private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName()); private static final Long ACTIONS = 100L; @@ -41,14 +33,14 @@ class ClientTest { private final TestExtension.Helper helper; - ClientTest(TestExtension.Helper helper) { + BulkClientTest(TestExtension.Helper helper) { this.helper = helper; } @Test void testSingleDoc() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) @@ -58,8 +50,6 @@ class ClientTest { client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { @@ -72,8 +62,8 @@ class ClientTest { @Test void testNewIndex() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); @@ -83,8 +73,8 @@ class ClientTest { @Test void testMapping() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); @@ -98,20 +88,16 @@ class ClientTest { .endObject() .endObject() .endObject(); - client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); + client.newIndex("test", Settings.EMPTY, builder); + assertTrue(client.getMapping("test", "doc").containsKey("properties")); client.close(); } @Test void testRandomDocs() throws Exception { long numactions = ACTIONS; - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) @@ -123,10 +109,6 @@ class ClientTest { } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); } finally { assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { @@ -134,10 +116,7 @@ class ClientTest { } assertNull(client.getBulkController().getLastBulkError()); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(numactions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); + assertEquals(numactions, client.getSearchableDocs("test")); client.close(); } } @@ -146,10 +125,10 @@ class ClientTest { void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final Long actions = ACTIONS; + final long actions = ACTIONS; logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) @@ -160,15 +139,15 @@ class ClientTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test", settings, (String)null) - .startBulk("test", 0, 1000); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); logger.info("index created"); ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { executorService.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}"); + bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); @@ -176,8 +155,8 @@ class ClientTest { logger.info("waiting for latch..."); if (latch.await(60L, TimeUnit.SECONDS)) { logger.info("flush..."); - client.flush(); - client.waitForResponses(60L, TimeUnit.SECONDS); + bulkClient.flush(); + bulkClient.waitForResponses(60L, TimeUnit.SECONDS); logger.info("got all responses, executor service shutdown..."); executorService.shutdown(); executorService.awaitTermination(60L, TimeUnit.SECONDS); @@ -185,23 +164,16 @@ class ClientTest { } else { logger.warn("latch timeout"); } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); } finally { - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(maxthreads * actions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.close(); + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); } } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java index 5e966c0..44cff60 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java @@ -2,19 +2,15 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -24,9 +20,9 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + private static final Long ACTIONS = 100L; - private static final Long ACTIONS = 50L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; private final TestExtension.Helper helper; @@ -37,8 +33,8 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpBulkClient client = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); @@ -51,15 +47,8 @@ class DuplicateIDTest { client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test") - .setTypes("test") - .setQuery(matchAllQuery()); - long hits = searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); - logger.info("hits = {}", hits); + long hits = client.getSearchableDocs("test"); assertTrue(hits < ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { client.close(); assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java index 54c3d6f..207953f 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,50 +35,49 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1")) + final HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) + .put(helper.getHttpSettings()) + .build(); + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) - .provider(ExtendedHttpClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1", settings); - client.shiftIndex("test", "test1", Collections.emptyList()); - client.newIndex("test2", settings); - client.shiftIndex("test", "test2", Collections.emptyList()); - client.newIndex("test3", settings); - client.shiftIndex("test", "test3", Collections.emptyList()); - client.newIndex("test4", settings); - client.shiftIndex("test", "test4", Collections.emptyList()); + bulkClient.newIndex("test1", settings); + adminClient.shiftIndex("test", "test1", Collections.emptyList()); + bulkClient.newIndex("test2", settings); + adminClient.shiftIndex("test", "test2", Collections.emptyList()); + bulkClient.newIndex("test3", settings); + adminClient.shiftIndex("test", "test3", Collections.emptyList()); + bulkClient.newIndex("test4", settings); + adminClient.shiftIndex("test", "test4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test", "test4", 2, 2, true); + adminClient.pruneIndex("test", "test4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test4")); List list = new ArrayList<>(); for (String index : Arrays.asList("test1", "test2", "test3", "test4")) { - IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(index); - IndicesExistsResponse indicesExistsResponse = - client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); - list.add(indicesExistsResponse.isExists()); + list.add(bulkClient.isIndexExists(index)); } logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.close(); } } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java index 234c0de..25ca051 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java @@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -35,56 +35,54 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() + final HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) + .put(helper.getHttpSettings()) + .build(); + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) .put(helper.getHttpSettings()) - .provider(ExtendedHttpClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1234", settings); + bulkClient.newIndex("test1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", helper.randomString(1), false, + bulkClient.index("test1234", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); - + adminClient.shiftIndex("test", "test1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - - Map aliases = client.getAliases("test1234"); + Map aliases = adminClient.getAliases("test1234"); logger.log(Level.DEBUG, "aliases = " + aliases); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - String resolved = client.resolveAlias("test"); + String resolved = adminClient.resolveAlias("test"); logger.log(Level.DEBUG, "resolved = " + resolved); - aliases = client.getAliases(resolved); + aliases = adminClient.getAliases(resolved); logger.log(Level.DEBUG, "aliases = " + aliases); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - client.newIndex("test5678", settings); + bulkClient.newIndex("test5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", helper.randomString(1), false, + bulkClient.index("test5678", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - - indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -94,31 +92,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - - aliases = client.getAliases("test5678"); + aliases = adminClient.getAliases("test5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - resolved = client.resolveAlias("test"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + adminClient.close(); } } } diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java index b2b86fb..d2663f5 100644 --- a/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; +import org.xbib.elx.http.HttpAdminClient; +import org.xbib.elx.http.HttpAdminClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,47 +30,51 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) + final HttpAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(HttpAdminClientProvider.class) .put(helper.getHttpSettings()) .build(); + final HttpBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(HttpBulkClientProvider.class) + .put(helper.getHttpSettings()) + .build(); + IndexDefinition indexDefinition = adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() + .build()); try { - assertEquals(helper.getClusterName(), client.getClusterName()); - client.newIndex("test_smoke"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); - client.delete("test_smoke", "1"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() - .build()); + assertEquals(helper.getClusterName(), adminClient.getClusterName()); + bulkClient.newIndex("test_smoke"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); + bulkClient.delete("test_smoke", "1"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + adminClient.deleteIndex("test_smoke"); assertEquals(0, indexDefinition.getReplicaLevel()); - client.newIndex(indexDefinition); - client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); + bulkClient.newIndex(indexDefinition); + bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.updateReplicaLevel(indexDefinition, 2); + int replica = adminClient.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkMetric().getFailed().getCount()); - assertEquals(6, client.getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + // close admin after bulk + adminClient.deleteIndex(indexDefinition); + adminClient.close(); } } } diff --git a/elx-http/src/test/resources/log4j2.xml b/elx-http/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-http/src/test/resources/log4j2.xml rename to elx-http/src/test/resources/log4j2-test.xml diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java deleted file mode 100644 index 46a4e9a..0000000 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.node; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedNodeClientProvider implements ExtendedClientProvider { - @Override - public ExtendedNodeClient getExtendedClient() { - return new ExtendedNodeClient(); - } -} diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java similarity index 91% rename from elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java rename to elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java index fab3714..df8d9d0 100644 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java @@ -11,15 +11,15 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; -import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.elx.common.AbstractAdminClient; import java.io.IOException; import java.util.Collection; import java.util.Collections; -public class ExtendedNodeClient extends AbstractExtendedClient { +public class NodeAdminClient extends AbstractAdminClient { - private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName()); + private static final Logger logger = LogManager.getLogger(NodeAdminClient.class.getName()); private Node node; diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java new file mode 100644 index 0000000..ec949e0 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.AdminClientProvider; + +public class NodeAdminClientProvider implements AdminClientProvider { + + @Override + public NodeAdminClient getClient() { + return new NodeAdminClient(); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java new file mode 100644 index 0000000..d469609 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java @@ -0,0 +1,68 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class NodeBulkClient extends AbstractBulkClient { + + private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName()); + + private Node node; + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings == null) { + return null; + } + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); + logger.info("creating node client on {} with effective settings {}", + version, Strings.toString(builder)); + Collection> plugins = Collections.emptyList(); + this.node = new BulkNode(new Environment(effectiveSettings, null), plugins); + try { + node.start(); + } catch (Exception e) { + throw new IOException(e); + } + return node.client(); + } + + @Override + protected void closeClient() throws IOException { + if (node != null) { + logger.debug("closing node client"); + node.close(); + } + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, classpathPlugins); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java new file mode 100644 index 0000000..f40a021 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.BulkClientProvider; + +public class NodeBulkClientProvider implements BulkClientProvider { + + @Override + public NodeBulkClient getClient() { + return new NodeBulkClient(); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java new file mode 100644 index 0000000..ab7f35d --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java @@ -0,0 +1,68 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class NodeSearchClient extends AbstractSearchClient { + + private static final Logger logger = LogManager.getLogger(NodeSearchClient.class.getName()); + + private Node node; + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings == null) { + return null; + } + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); + logger.info("creating node client on {} with effective settings {}", + version, Strings.toString(builder)); + Collection> plugins = Collections.emptyList(); + this.node = new BulkNode(new Environment(effectiveSettings, null), plugins); + try { + node.start(); + } catch (Exception e) { + throw new IOException(e); + } + return node.client(); + } + + @Override + protected void closeClient() throws IOException { + if (node != null) { + logger.debug("closing node client"); + node.close(); + } + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, classpathPlugins); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java new file mode 100644 index 0000000..233c2a4 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.SearchClientProvider; + +public class NodeSearchClientProvider implements SearchClientProvider { + + @Override + public NodeSearchClient getClient() { + return new NodeSearchClient(); + } +} diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..acc3b0c --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeAdminClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..62630f5 --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeBulkClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 372aaad..0000000 --- a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.node.ExtendedNodeClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..54b437c --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeSearchClientProvider \ No newline at end of file diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java new file mode 100644 index 0000000..16b2c69 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java @@ -0,0 +1,174 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName()); + + private static final Long ACTIONS = 10000L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + bulkClient.newIndex("test"); + bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + bulkClient.newIndex("test"); + bulkClient.close(); + } + + @Test + void testMapping() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .build(); + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties")); + bulkClient.close(); + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final long actions = ACTIONS; + logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(30L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(30L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java deleted file mode 100644 index d071e40..0000000 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ /dev/null @@ -1,200 +0,0 @@ -package org.xbib.elx.node.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.Parameters; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@ExtendWith(TestExtension.class) -class ClientTest { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - - private static final Long ACTIONS = 10000L; - - private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; - - private final TestExtension.Helper helper; - - ClientTest(TestExtension.Helper helper) { - this.helper = helper; - } - - @Test - void testSingleDoc() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .build(); - try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - void testNewIndex() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - client.newIndex("test"); - client.close(); - } - - @Test - void testMapping() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - XContentBuilder builder = JsonXContent.contentBuilder() - .startObject() - .startObject("doc") - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject() - .endObject(); - client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); - client.close(); - } - - @Test - void testRandomDocs() throws Exception { - long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test"); - for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(numactions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.close(); - } - } - - @Test - void testThreadedRandomDocs() throws Exception { - int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final long actions = ACTIONS; - logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - Settings settings = Settings.builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - client.newIndex("test", settings, (String)null) - .startBulk("test", 0, 1000); - logger.info("index created"); - ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); - final CountDownLatch latch = new CountDownLatch(maxthreads); - for (int i = 0; i < maxthreads; i++) { - executorService.execute(() -> { - for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - latch.countDown(); - }); - } - logger.info("waiting for latch..."); - if (latch.await(30L, TimeUnit.SECONDS)) { - logger.info("flush..."); - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - logger.info("got all responses, executor service shutdown..."); - executorService.shutdown(); - executorService.awaitTermination(30L, TimeUnit.SECONDS); - logger.info("pool is shut down"); - } else { - logger.warn("latch timeout"); - } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setQuery(QueryBuilders.matchAllQuery()).setSize(0); - assertEquals(maxthreads * actions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.close(); - } - } -} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index 16b782f..7ba64f4 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -2,17 +2,12 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -25,9 +20,9 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + private static final Long ACTIONS = 100L; - private static final Long ACTIONS = 50L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; private final TestExtension.Helper helper; @@ -38,37 +33,27 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { - client.newIndex("test"); + bulkClient.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test", helper.randomString(1), false, + bulkClient.index("test", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.types("test"); - searchRequest.source(builder); - long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); - logger.info("hits = {}", hits); - assertTrue(hits < ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test"); + assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS); } finally { - client.close(); - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 43e87ea..8b2245f 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,49 +35,47 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_prune1", settings); - client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); - client.newIndex("test_prune2", settings); - client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); - client.newIndex("test_prune3", settings); - client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); - client.newIndex("test_prune4", settings); - client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + bulkClient.newIndex("test_prune1", settings); + adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + bulkClient.newIndex("test_prune2", settings); + adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + bulkClient.newIndex("test_prune3", settings); + adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + bulkClient.newIndex("test_prune4", settings); + adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test_prune", "test_prune4", 2, 2, true); + adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); List list = new ArrayList<>(); for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { - IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(index); - IndicesExistsResponse indicesExistsResponse = - client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); - list.add(indicesExistsResponse.isExists()); + list.add(bulkClient.isIndexExists(index)); } logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index fb7aaaf..e30dd92 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -3,15 +3,16 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -33,50 +34,49 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_shift", settings); + bulkClient.newIndex("test_shift", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift", helper.randomString(1), false, + bulkClient.index("test_shift", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); + adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - - Map aliases = client.getAliases("test_shift"); + Map aliases = adminClient.getAliases("test_shift"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - String resolved = client.resolveAlias("test"); - aliases = client.getAliases(resolved); + String resolved = adminClient.resolveAlias("test"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test")); - - client.newIndex("test_shift2", settings); + bulkClient.newIndex("test_shift2", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift2", helper.randomString(1), false, + bulkClient.index("test_shift2", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - - indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -86,32 +86,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - - aliases = client.getAliases("test_shift2"); + aliases = adminClient.getAliases("test_shift2"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - resolved = client.resolveAlias("test"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index d6e458d..c802cd0 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.api.IndexDefinition; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,46 +30,49 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .build(); + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); try { - assertEquals(helper.getClusterName(), client.getClusterName()); - client.newIndex("test_smoke"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() - .build()); + assertEquals(helper.getClusterName(), adminClient.getClusterName()); + bulkClient.newIndex("test_smoke"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + adminClient.deleteIndex("test_smoke"); assertEquals(0, indexDefinition.getReplicaLevel()); - client.newIndex(indexDefinition); - client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); + bulkClient.newIndex(indexDefinition); + bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.updateReplicaLevel(indexDefinition, 2); + int replica = adminClient.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - //assertEquals(0, client.getBulkMetric().getFailed().getCount()); - //assertEquals(6, client.getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + // close admin after bulk + adminClient.deleteIndex(indexDefinition); + adminClient.close(); } } } diff --git a/elx-node/src/test/resources/log4j2.xml b/elx-node/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-node/src/test/resources/log4j2.xml rename to elx-node/src/test/resources/log4j2-test.xml diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java deleted file mode 100644 index 669d21a..0000000 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.xbib.elx.transport; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedTransportClientProvider implements ExtendedClientProvider { - - @Override - public ExtendedTransportClient getExtendedClient() { - return new ExtendedTransportClient(); - } -} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java new file mode 100644 index 0000000..3987f63 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java @@ -0,0 +1,40 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractAdminClient; + +import java.io.IOException; + +/** + * Transport admin client. + */ +public class TransportAdminClient extends AbstractAdminClient { + + private final TransportClientHelper helper; + + public TransportAdminClient() { + this.helper = new TransportClientHelper(); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java new file mode 100644 index 0000000..18d2368 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.AdminClientProvider; + +public class TransportAdminClientProvider implements AdminClientProvider { + + @Override + public TransportAdminClient getClient() { + return new TransportAdminClient(); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java new file mode 100644 index 0000000..ebc8621 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java @@ -0,0 +1,39 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +/** + * Transport search client with additional methods. + */ +public class TransportBulkClient extends AbstractBulkClient { + + private final TransportClientHelper helper; + + public TransportBulkClient() { + this.helper = new TransportClientHelper(); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java new file mode 100644 index 0000000..399d73d --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.BulkClientProvider; + +public class TransportBulkClientProvider implements BulkClientProvider { + + @Override + public TransportBulkClient getClient() { + return new TransportBulkClient(); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java similarity index 77% rename from elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java rename to elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java index b5b3a51..7910c3a 100644 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java @@ -23,9 +23,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.transport.netty4.Netty4Plugin; -import org.xbib.elx.common.AbstractExtendedClient; import org.xbib.elx.common.util.NetworkUtils; - import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; @@ -33,14 +31,11 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -/** - * Transport client with additional methods using the BulkProcessor. - */ -public class ExtendedTransportClient extends AbstractExtendedClient { +public class TransportClientHelper { - private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName()); - @Override + private static final Logger logger = LogManager.getLogger(TransportAdminClient.class.getName()); + protected ElasticsearchClient createClient(Settings settings) throws IOException { if (settings != null) { String systemIdentifier = System.getProperty("os.name") @@ -49,11 +44,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient { + " " + System.getProperty("java.vm.version") + " Elasticsearch " + Version.CURRENT.toString(); Settings transportClientSettings = getTransportClientSettings(settings); - //XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject(); XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject(); logger.log(Level.INFO, "creating transport client on {} with settings {}", systemIdentifier, - //Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()), Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder, ToXContent.EMPTY_PARAMS).endObject())); return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class)); @@ -61,31 +54,14 @@ public class ExtendedTransportClient extends AbstractExtendedClient { return null; } - @Override - protected void closeClient() { - if (getClient() != null) { - TransportClient client = (TransportClient) getClient(); - client.close(); - client.threadPool().shutdown(); + public void init(TransportClient transportClient, Settings settings) throws IOException { + Collection addrs = findAddresses(settings); + if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.toString()); } } - @Override - public ExtendedTransportClient init(Settings settings) throws IOException { - super.init(settings); - // additional auto-connect - try { - Collection addrs = findAddresses(settings); - if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { - throw new NoNodeAvailableException("no cluster nodes available, check settings " - + settings.toString()); - } - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - return this; - } - private Collection findAddresses(Settings settings) throws IOException { final int defaultPort = settings.getAsInt("port", 9300); Collection addresses = new ArrayList<>(); @@ -112,11 +88,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient { return addresses; } - private boolean connect(Collection addresses, boolean autodiscover) { - if (getClient() == null) { - throw new IllegalStateException("no client present"); - } - TransportClient transportClient = (TransportClient) getClient(); + private boolean connect(TransportClient transportClient, Collection addresses, boolean autodiscover) { for (TransportAddress address : addresses) { transportClient.addTransportAddresses(address); } @@ -126,7 +98,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient { if (autodiscover) { logger.debug("trying to auto-discover all nodes..."); ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); + new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE); ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); addDiscoveryNodes(transportClient, discoveryNodes); diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java new file mode 100644 index 0000000..750914c --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java @@ -0,0 +1,39 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +/** + * Transport search client with additional methods. + */ +public class TransportSearchClient extends AbstractSearchClient { + + private final TransportClientHelper helper; + + public TransportSearchClient() { + this.helper = new TransportClientHelper(); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java new file mode 100644 index 0000000..c77e320 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.SearchClientProvider; + +public class TransportSearchClientProvider implements SearchClientProvider { + + @Override + public TransportSearchClient getClient() { + return new TransportSearchClient(); + } +} diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..37ce7ac --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportAdminClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..e1c69bd --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportBulkClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 640e2f9..0000000 --- a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.transport.ExtendedTransportClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..ae6a33b --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportSearchClientProvider \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java new file mode 100644 index 0000000..388f6f3 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java @@ -0,0 +1,188 @@ +package org.xbib.elx.transport.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName()); + + private static final Long ACTIONS = 10000L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + bulkClient.newIndex("test"); + bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + bulkClient.newIndex("test"); + bulkClient.close(); + } + + @Test + void testMapping() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties")); + bulkClient.close(); + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final long actions = ACTIONS; + logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .put(Parameters.ENABLE_BULK_LOGGING.name(), "true") + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + logger.info("index created"); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(30L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(30L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java deleted file mode 100644 index 76e6bf5..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ /dev/null @@ -1,223 +0,0 @@ -package org.xbib.elx.transport.test; - -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.Parameters; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@ExtendWith(TestExtension.class) -class ClientTest { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - - private static final Long ACTIONS = 10000L; - - private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; - - private final TestExtension.Helper helper; - - ClientTest(TestExtension.Helper helper) { - this.helper = helper; - } - - @Test - void testSingleDoc() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .build(); - try { - client.newIndex("test"); - client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - void testNewIndex() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - client.newIndex("test"); - client.close(); - } - - @Test - void testMapping() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - XContentBuilder builder = JsonXContent.contentBuilder() - .startObject() - .startObject("doc") - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject() - .endObject(); - client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); - client.close(); - } - - @Test - void testRandomDocs() throws Exception { - long numactions = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test"); - for (int i = 0; i < ACTIONS; i++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.source(builder); - SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - logger.log(Level.INFO, searchResponse.toString()); - assertEquals(numactions, searchResponse.getHits().getTotalHits()); - client.close(); - } - } - - @Test - void testThreadedRandomDocs() throws Exception { - int maxthreads = Runtime.getRuntime().availableProcessors(); - Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final long actions = ACTIONS; - logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .put(Parameters.ENABLE_BULK_LOGGING.name(), "true") - .build(); - try { - Settings settings = Settings.builder() - .put("index.number_of_shards", 1) - .put("index.number_of_replicas", 0) - .build(); - client.newIndex("test", settings, (String)null) - .startBulk("test", 0, 1000); - logger.info("index created"); - ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); - final CountDownLatch latch = new CountDownLatch(maxthreads); - for (int i = 0; i < maxthreads; i++) { - executorService.execute(() -> { - for (int i1 = 0; i1 < actions; i1++) { - client.index("test", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - latch.countDown(); - }); - } - logger.info("waiting for latch..."); - if (latch.await(30L, TimeUnit.SECONDS)) { - logger.info("flush..."); - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - logger.info("got all responses, executor service shutdown..."); - executorService.shutdown(); - executorService.awaitTermination(30L, TimeUnit.SECONDS); - logger.info("pool is shut down"); - } else { - logger.warn("latch timeout"); - } - client.stopBulk("test", 30L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } catch (Exception e) { - logger.error(e.getMessage(), e); - } finally { - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.source(builder); - SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet(); - logger.log(Level.INFO, searchResponse.toString()); - assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits()); - client.close(); - } - } -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index c1afc40..ffd23d2 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -2,18 +2,12 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -26,9 +20,9 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + private static final Long ACTIONS = 100L; - private static final Long ACTIONS = 5L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; private final TestExtension.Helper helper; @@ -39,41 +33,28 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(helper.getTransportSettings()) .build(); try { - client.newIndex("test_dup"); + bulkClient.newIndex("test_dup"); for (int i = 0; i < ACTIONS; i++) { - client.index("test_dup", helper.randomString(1), false, + bulkClient.index("test_dup", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test_dup"); - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - builder.size(0); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test_dup"); - searchRequest.types("test_dup"); - searchRequest.source(builder); - SearchResponse searchResponse = - helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits(); - logger.info("hits = {}", hits); - assertTrue(hits < ACTIONS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.refreshIndex("test_dup"); + assertTrue(bulkClient.getSearchableDocs("test_dup") < ACTIONS); } finally { - client.close(); - assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index 8d82ba3..0640fbd 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,8 +35,12 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(helper.getTransportSettings()) .build(); try { @@ -46,41 +48,36 @@ class IndexPruneTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_prune1", settings); - client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); - client.newIndex("test_prune2", settings); - client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); - client.newIndex("test_prune3", settings); - client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); - client.newIndex("test_prune4", settings); - client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + bulkClient.newIndex("test_prune1", settings); + adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + bulkClient.newIndex("test_prune2", settings); + adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + bulkClient.newIndex("test_prune3", settings); + adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + bulkClient.newIndex("test_prune4", settings); + adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test_prune", "test_prune4", 2, 2, true); + adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); List list = new ArrayList<>(); for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { - IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(index); - IndicesExistsResponse indicesExistsResponse = - client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); - list.add(indicesExistsResponse.isExists()); + list.add(bulkClient.isIndexExists(index)); } logger.info(list); assertFalse(list.get(0)); assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index f182956..bf59064 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -3,15 +3,16 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -33,8 +34,12 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(helper.getTransportSettings()) .build(); try { @@ -42,38 +47,38 @@ class IndexShiftTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_shift1234", settings); + bulkClient.newIndex("test_shift1234", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift1234", helper.randomString(1), false, + bulkClient.index("test_shift1234", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c")); + adminClient.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - Map aliases = client.getAliases("test_shift1234"); + Map aliases = adminClient.getAliases("test_shift1234"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test_shift")); - String resolved = client.resolveAlias("test_shift"); - aliases = client.getAliases(resolved); + String resolved = adminClient.resolveAlias("test_shift"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test_shift")); - client.newIndex("test_shift5678", settings); + bulkClient.newIndex("test_shift5678", settings); for (int i = 0; i < 1; i++) { - client.index("test_shift5678", helper.randomString(1), false, + bulkClient.index("test_shift5678", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = client.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"), + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + indexShiftResult = adminClient.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"), (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); @@ -83,29 +88,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - aliases = client.getAliases("test_shift5678"); + aliases = adminClient.getAliases("test_shift5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - resolved = client.resolveAlias("test_shift"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test_shift"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index ac17b60..b005f12 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,44 +30,48 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) .put(helper.getTransportSettings()) .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); try { - assertEquals(helper.getClusterName(), client.getClusterName()); - client.newIndex("test_smoke"); - client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.delete("test_smoke", "1"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.checkMapping("test_smoke"); - client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() - .build()); + assertEquals(helper.getClusterName(), adminClient.getClusterName()); + bulkClient.newIndex("test_smoke"); + bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + bulkClient.delete("test_smoke", "1"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.checkMapping("test_smoke"); + adminClient.deleteIndex("test_smoke"); assertEquals(0, indexDefinition.getReplicaLevel()); - client.newIndex(indexDefinition); - client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30, TimeUnit.SECONDS); - client.updateReplicaLevel(indexDefinition, 2); - int replica = client.getReplicaLevel(indexDefinition); + bulkClient.newIndex(indexDefinition); + bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); + bulkClient.flush(); + bulkClient.waitForResponses(30, TimeUnit.SECONDS); + adminClient.updateReplicaLevel(indexDefinition, 2); + int replica = adminClient.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkMetric().getFailed().getCount()); - assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(4, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + // close admin after bulk + adminClient.deleteIndex(indexDefinition); + adminClient.close(); } } }