diff --git a/build.gradle b/build.gradle index 3f71fc8..6f4353e 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ wrapper { ext { user = 'jprante' name = 'elx' - description = 'Extensions for Elasticsearch clients (node and transport)' + description = 'Admin/Bulk/Search API extensions for Elasticsearch clients (node, transport, http)' inceptionYear = '2019' url = 'https://github.com/' + user + '/' + name scmUrl = 'https://github.com/' + user + '/' + name diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java new file mode 100644 index 0000000..bdb5b5d --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClient.java @@ -0,0 +1,194 @@ +package org.xbib.elx.api; + +import org.elasticsearch.common.settings.Settings; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +/** + * Interface for extended managing and indexing methods of an Elasticsearch client. + */ +public interface AdminClient extends NativeClient { + + /** + * Build index definition from settings. + * + * @param index the index name + * @param settings the settings for the index + * @return index definition + * @throws IOException if settings/mapping URL is invalid/malformed + */ + IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; + + /** + * Delete an index. + * @param indexDefinition the index definition + * @return this + */ + AdminClient deleteIndex(IndexDefinition indexDefinition); + + /** + * Delete an index. + * + * @param index index + * @return this + */ + AdminClient deleteIndex(String index); + + + /** + * Update replica level. + * @param indexDefinition the index definition + * @param level the replica level + * @return this + * @throws IOException if replica setting could not be updated + */ + AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; + + /** + * Update replica level. + * + * @param index index + * @param level the replica level + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return this + * @throws IOException if replica setting could not be updated + */ + AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; + + /** + * Get replica level. + * @param indexDefinition the index name + * @return the replica level of the index + */ + int getReplicaLevel(IndexDefinition indexDefinition); + + /** + * Get replica level. + * @param index the index name + * @return the replica level of the index + */ + int getReplicaLevel(String index); + + /** + * Force segment merge of an index. + * @param indexDefinition the index definition + * @return this + */ + boolean forceMerge(IndexDefinition indexDefinition); + + /** + * Force segment merge of an index. + * @param index the index + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return this + */ + boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); + + + /** + * Wait for index recovery (after replica change). + * + * @param index index + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return true if wait succeeded, false if wait timed out + */ + boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit); + + /** + * Resolve alias. + * + * @param alias the alias + * @return this index name behind the alias or the alias if there is no index + */ + String resolveAlias(String alias); + + /** + * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index + * name. + * + * @param alias the alias + * @return the most recent index name pointing to the alias + */ + String resolveMostRecentIndex(String alias); + + /** + * Get all index aliases. + * @param index the index + * @return map of index aliases + */ + Map getAliases(String index); + + /** + * Shift from one index to another. + * @param indexDefinition the index definition + * @param additionalAliases new aliases + * @return this + */ + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); + + /** + * Shift from one index to another. + * @param indexDefinition the index definition + * @param additionalAliases new aliases + * @param indexAliasAdder method to add aliases + * @return this + */ + IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, + IndexAliasAdder indexAliasAdder); + + /** + * Shift from one index to another. + * @param index the index name + * @param fullIndexName the index name with timestamp + * @param additionalAliases a list of names that should be set as index aliases + * @return this + */ + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); + + /** + * Shift from one index to another. + * @param index the index name + * @param fullIndexName the index name with timestamp + * @param additionalAliases a list of names that should be set as index aliases + * @param adder an adder method to create alias term queries + * @return this + */ + IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, + IndexAliasAdder adder); + + /** + * Prune index. + * @param indexDefinition the index definition + * @return the index prune result + */ + IndexPruneResult pruneIndex(IndexDefinition indexDefinition); + + /** + * Apply retention policy to prune indices. All indices before delta should be deleted, + * but the number of mintokeep indices must be kept. + * + * @param index index name + * @param fullIndexName index name with timestamp + * @param delta timestamp delta (for index timestamps) + * @param mintokeep minimum number of indices to keep + * @param perform true if pruning should be executed, false if not + * @return the index prune result + */ + IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); + + /** + * Find the timestamp of the most recently indexed document in the index. + * + * @param index the index name + * @param timestampfieldname the timestamp field name + * @return millis UTC millis of the most recent document + * @throws IOException if most rcent document can not be found + */ + Long mostRecentDocument(String index, String timestampfieldname) throws IOException; +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java new file mode 100644 index 0000000..954efd5 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface AdminClientProvider { + + C getClient(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java new file mode 100644 index 0000000..5f21d14 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClient.java @@ -0,0 +1,227 @@ +package org.xbib.elx.api; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.Flushable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public interface BulkClient extends NativeClient, Flushable { + + /** + * Get bulk metric. + * @return the bulk metric + */ + BulkMetric getBulkMetric(); + + /** + * Get buulk control. + * @return the bulk control + */ + BulkController getBulkController(); + + + /** + * Create a new index. + * + * @param index index + * @throws IOException if new index creation fails + */ + void newIndex(String index) throws IOException; + + /** + * Create a new index. + * @param indexDefinition the index definition + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(IndexDefinition indexDefinition) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @throws IOException if settings is invalid or index creation fails + */ + void newIndex(String index, Settings settings) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; + + /** + * Create a new index. + * + * @param index index + * @param settings settings + * @param mapping mapping + * @throws IOException if settings/mapping is invalid or index creation fails + */ + void newIndex(String index, Settings settings, Map mapping) throws IOException; + + /** + * Add index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. + * + * @param index the index + * @param id the id + * @param create true if document must be created + * @param source the source + * @return this + */ + BulkClient index(String index, String id, boolean create, BytesReference source); + + /** + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when limits are exceeded. + * + * @param index the index + * @param id the id + * @param create true if document is to be created, false otherwise + * @param source the source + * @return this client methods + */ + BulkClient index(String index, String id, boolean create, String source); + + /** + * Index request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param indexRequest the index request to add + * @return this + */ + BulkClient index(IndexRequest indexRequest); + + /** + * Delete request. + * + * @param index the index + * @param id the id + * @return this + */ + BulkClient delete(String index, String id); + + /** + * Delete request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * + * @param deleteRequest the delete request to add + * @return this + */ + BulkClient delete(DeleteRequest deleteRequest); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param index the index + * @param id the id + * @param source the source + * @return this + */ + BulkClient update(String index, String id, BytesReference source); + + /** + * Update document. Use with precaution! Does not work in all cases. + * + * @param index the index + * @param id the id + * @param source the source + * @return this + */ + BulkClient update(String index, String id, String source); + + /** + * Bulked update request. Each request will be added to a queue for bulking requests. + * Submitting request will be done when bulk limits are exceeded. + * Note that updates only work correctly when all operations between nodes are synchronized. + * + * @param updateRequest the update request to add + * @return this + */ + BulkClient update(UpdateRequest updateRequest); + + /** + * Start bulk mode for indexes. + * @param indexDefinition index definition + * @throws IOException if bulk could not be started + */ + void startBulk(IndexDefinition indexDefinition) throws IOException; + + /** + * Start bulk mode. + * + * @param index index + * @param startRefreshIntervalSeconds refresh interval before bulk + * @param stopRefreshIntervalSeconds refresh interval after bulk + * @throws IOException if bulk could not be started + */ + void startBulk(String index, long startRefreshIntervalSeconds, + long stopRefreshIntervalSeconds) throws IOException; + + + /** + * Stop bulk mode. + * + * @param indexDefinition index definition + * @throws IOException if bulk could not be startet + */ + void stopBulk(IndexDefinition indexDefinition) throws IOException; + + /** + * Stops bulk mode. + * + * @param index index + * @param timeout maximum wait time + * @param timeUnit time unit for timeout + * @throws IOException if bulk could not be stopped + */ + void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; + + /** + * Wait for all outstanding bulk responses. + * + * @param timeout maximum wait time + * @param timeUnit unit of timeout value + * @return true if wait succeeded, false if wait timed out + */ + boolean waitForResponses(long timeout, TimeUnit timeUnit); + + /** + * Update index setting. + * @param index the index + * @param key the key of the value to be updated + * @param value the new value + * @param timeout timeout + * @param timeUnit time unit + * @throws IOException if update index setting failed + */ + void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; + + /** + * Refresh the index. + * + * @param index index + */ + void refreshIndex(String index); + + /** + * Flush the index. The cluster clears cache and completes indexing. + * + * @param index index + */ + void flushIndex(String index); + +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java new file mode 100644 index 0000000..0f013e2 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface BulkClientProvider { + + C getClient(); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java deleted file mode 100644 index d6c761e..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java +++ /dev/null @@ -1,487 +0,0 @@ -package org.xbib.elx.api; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * Interface for extended managing and indexing methods of an Elasticsearch client. - */ -public interface ExtendedClient extends Flushable, Closeable { - - /** - * Set an Elasticsearch client to extend from it. May be null for TransportClient. - * @param client client - * @return this client - */ - ExtendedClient setClient(ElasticsearchClient client); - - /** - * Return Elasticsearch client. - * - * @return Elasticsearch client - */ - ElasticsearchClient getClient(); - - /** - * Get buulk control. - * @return the bulk control - */ - BulkController getBulkController(); - - /** - * Initiative the extended client, the bulk metric and bulk controller, - * creates instances and connect to cluster, if required. - * - * @param settings settings - * @return this client - * @throws IOException if init fails - */ - ExtendedClient init(Settings settings) throws IOException; - - /** - * Build index definition from settings. - * - * @param index the index name - * @param settings the settings for the index - * @return index definition - * @throws IOException if settings/mapping URL is invalid/malformed - */ - IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException; - - /** - * Add index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when limits are exceeded. - * - * @param index the index - * @param id the id - * @param create true if document must be created - * @param source the source - * @return this - */ - ExtendedClient index(String index, String id, boolean create, BytesReference source); - - /** - * Index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when limits are exceeded. - * - * @param index the index - * @param id the id - * @param create true if document is to be created, false otherwise - * @param source the source - * @return this client methods - */ - ExtendedClient index(String index, String id, boolean create, String source); - - /** - * Index request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param indexRequest the index request to add - * @return this - */ - ExtendedClient index(IndexRequest indexRequest); - - /** - * Delete request. - * - * @param index the index - * @param id the id - * @return this - */ - ExtendedClient delete(String index, String id); - - /** - * Delete request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * - * @param deleteRequest the delete request to add - * @return this - */ - ExtendedClient delete(DeleteRequest deleteRequest); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized. - * - * @param index the index - * @param id the id - * @param source the source - * @return this - * @throws IOException if update fails - */ - ExtendedClient update(String index, String id, BytesReference source) throws IOException; - - /** - * Update document. Use with precaution! Does not work in all cases. - * - * @param index the index - * @param id the id - * @param source the source - * @return this - */ - ExtendedClient update(String index, String id, String source); - - /** - * Bulked update request. Each request will be added to a queue for bulking requests. - * Submitting request will be done when bulk limits are exceeded. - * Note that updates only work correctly when all operations between nodes are synchronized. - * - * @param updateRequest the update request to add - * @return this - */ - ExtendedClient update(UpdateRequest updateRequest); - - /** - * Create a new index. - * - * @param index index - * @return this - * @throws IOException if new index creation fails - */ - ExtendedClient newIndex(String index) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @return this - * @throws IOException if settings is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException; - - /** - * Create a new index. - * - * @param index index - * @param settings settings - * @param mapping mapping - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException; - - /** - * Create a new index. - * @param indexDefinition the index definition - * @return this - * @throws IOException if settings/mapping is invalid or index creation fails - */ - ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException; - - /** - * Delete an index. - * @param indexDefinition the index definition - * @return this - */ - ExtendedClient deleteIndex(IndexDefinition indexDefinition); - - /** - * Delete an index. - * - * @param index index - * @return this - */ - ExtendedClient deleteIndex(String index); - - /** - * Start bulk mode for indexes. - * @param indexDefinition index definition - * @return this - * @throws IOException if bulk could not be started - */ - ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException; - - /** - * Start bulk mode. - * - * @param index index - * @param startRefreshIntervalSeconds refresh interval before bulk - * @param stopRefreshIntervalSeconds refresh interval after bulk - * @return this - * @throws IOException if bulk could not be started - */ - ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, - long stopRefreshIntervalSeconds) throws IOException; - - /** - * Stop bulk mode. - * - * @param indexDefinition index definition - * @return this - * @throws IOException if bulk could not be startet - */ - ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException; - - /** - * Stops bulk mode. - * - * @param index index - * @param timeout maximum wait time - * @param timeUnit time unit for timeout - * @return this - * @throws IOException if bulk could not be stopped - */ - ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException; - - /** - * Update replica level. - * @param indexDefinition the index definition - * @param level the replica level - * @return this - * @throws IOException if replica setting could not be updated - */ - ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException; - - /** - * Update replica level. - * - * @param index index - * @param level the replica level - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return this - * @throws IOException if replica setting could not be updated - */ - ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException; - - /** - * Get replica level. - * @param indexDefinition the index name - * @return the replica level of the index - */ - int getReplicaLevel(IndexDefinition indexDefinition); - - /** - * Get replica level. - * @param index the index name - * @return the replica level of the index - */ - int getReplicaLevel(String index); - - /** - * Refresh the index. - * - * @param index index - * @return this - */ - ExtendedClient refreshIndex(String index); - - /** - * Flush the index. The cluster clears cache and completes indexing. - * - * @param index index - * @return this - */ - ExtendedClient flushIndex(String index); - - /** - * Force segment merge of an index. - * @param indexDefinition the index definition - * @return this - */ - boolean forceMerge(IndexDefinition indexDefinition); - - /** - * Force segment merge of an index. - * @param index the index - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return this - */ - boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit); - - /** - * Wait for all outstanding bulk responses. - * - * @param timeout maximum wait time - * @param timeUnit unit of timeout value - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForResponses(long timeout, TimeUnit timeUnit); - - /** - * Wait for cluster being healthy. - * - * @param healthColor cluster health color to wait for - * @param maxWaitTime time value - * @param timeUnit time unit - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); - - /** - * Get current health color. - * - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return the cluster health color - */ - String getHealthColor(long maxWaitTime, TimeUnit timeUnit); - - /** - * Wait for index recovery (after replica change). - * - * @param index index - * @param maxWaitTime maximum wait time - * @param timeUnit time unit - * @return true if wait succeeded, false if wait timed out - */ - boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit); - - /** - * Update index setting. - * @param index the index - * @param key the key of the value to be updated - * @param value the new value - * @param timeout timeout - * @param timeUnit time unit - * @throws IOException if update index setting failed - */ - void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException; - - /** - * Resolve alias. - * - * @param alias the alias - * @return this index name behind the alias or the alias if there is no index - */ - String resolveAlias(String alias); - - /** - * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index - * name. - * - * @param alias the alias - * @return the most recent index name pointing to the alias - */ - String resolveMostRecentIndex(String alias); - - /** - * Get all index aliases. - * @param index the index - * @return map of index aliases - */ - Map getAliases(String index); - - /** - * Shift from one index to another. - * @param indexDefinition the index definition - * @param additionalAliases new aliases - * @return this - */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases); - - /** - * Shift from one index to another. - * @param indexDefinition the index definition - * @param additionalAliases new aliases - * @param indexAliasAdder method to add aliases - * @return this - */ - IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List additionalAliases, - IndexAliasAdder indexAliasAdder); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases); - - /** - * Shift from one index to another. - * @param index the index name - * @param fullIndexName the index name with timestamp - * @param additionalAliases a list of names that should be set as index aliases - * @param adder an adder method to create alias term queries - * @return this - */ - IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, - IndexAliasAdder adder); - - /** - * Prune index. - * @param indexDefinition the index definition - * @return the index prune result - */ - IndexPruneResult pruneIndex(IndexDefinition indexDefinition); - - /** - * Apply retention policy to prune indices. All indices before delta should be deleted, - * but the number of mintokeep indices must be kept. - * - * @param index index name - * @param fullIndexName index name with timestamp - * @param delta timestamp delta (for index timestamps) - * @param mintokeep minimum number of indices to keep - * @param perform true if pruning should be executed, false if not - * @return the index prune result - */ - IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform); - - /** - * Find the timestamp of the most recently indexed document in the index. - * - * @param index the index name - * @param timestampfieldname the timestamp field name - * @return millis UTC millis of the most recent document - * @throws IOException if most rcent document can not be found - */ - Long mostRecentDocument(String index, String timestampfieldname) throws IOException; - - /** - * Get cluster name. - * @return the cluster name - */ - String getClusterName(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java deleted file mode 100644 index 2a8904a..0000000 --- a/elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.xbib.elx.api; - -@FunctionalInterface -public interface ExtendedClientProvider { - - C getExtendedClient(); -} diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java index 49544a7..1c77ccb 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java @@ -1,7 +1,5 @@ package org.xbib.elx.api; -import java.net.MalformedURLException; -import java.net.URL; import java.util.concurrent.TimeUnit; public interface IndexDefinition { @@ -14,17 +12,13 @@ public interface IndexDefinition { String getFullIndexName(); - IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException; + IndexDefinition setSettings(String settings); - IndexDefinition setSettingsUrl(URL settingsUrl); + String getSettings(); - URL getSettingsUrl(); + IndexDefinition setMappings(String mappings); - IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException; - - IndexDefinition setMappingsUrl(URL mappingsUrl); - - URL getMappingsUrl(); + String getMappings(); IndexDefinition setDateTimePattern(String timeWindow); diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java index a4ef207..cd4c283 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java @@ -1,6 +1,6 @@ package org.xbib.elx.api; -import java.util.List; +import java.util.Collection; public interface IndexPruneResult { @@ -8,9 +8,9 @@ public interface IndexPruneResult { State getState(); - List getCandidateIndices(); + Collection getCandidateIndices(); - List getDeletedIndices(); + Collection getDeletedIndices(); boolean isAcknowledged(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java index 02a2e8c..6ef58d7 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java +++ b/elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java @@ -1,10 +1,10 @@ package org.xbib.elx.api; -import java.util.List; +import java.util.Collection; public interface IndexShiftResult { - List getMovedAliases(); + Collection getMovedAliases(); - List getNewAliases(); + Collection getNewAliases(); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/NativeClient.java b/elx-api/src/main/java/org/xbib/elx/api/NativeClient.java new file mode 100644 index 0000000..52fbbf3 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/NativeClient.java @@ -0,0 +1,63 @@ +package org.xbib.elx.api; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public interface NativeClient extends Closeable { + + /** + * Set an Elasticsearch client to extend from it. May be null for TransportClient. + * @param client the Elasticsearch client + */ + void setClient(ElasticsearchClient client); + + /** + * Return Elasticsearch client. + * + * @return Elasticsearch client + */ + ElasticsearchClient getClient(); + + /** + * Initiative the extended client, the bulk metric and bulk controller, + * creates instances and connect to cluster, if required. + * + * @param settings settings + * @throws IOException if init fails + */ + void init(Settings settings) throws IOException; + + /** + * Get cluster name. + * @return the cluster name + */ + String getClusterName(); + + /** + * Get current health color. + * + * @param maxWaitTime maximum wait time + * @param timeUnit time unit + * @return the cluster health color + */ + String getHealthColor(long maxWaitTime, TimeUnit timeUnit); + + /** + * Wait for cluster being healthy. + * + * @param healthColor cluster health color to wait for + * @param maxWaitTime time value + * @param timeUnit time unit + */ + void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit); + + Map getMapping(String index, String mapping); + + long getSearchableDocs(String index); + + boolean isIndexExists(String index); +} diff --git a/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java index 2ecb857..6c07bb2 100644 --- a/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java +++ b/elx-api/src/main/java/org/xbib/elx/api/ReadClient.java @@ -1,25 +1,27 @@ package org.xbib.elx.api; -import org.elasticsearch.action.ActionFuture; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.stream.Stream; -public interface ReadClient { +public interface SearchClient { - ActionFuture get(GetRequest getRequest); + Optional get(Consumer getRequestBuilder); - void get(GetRequest request, ActionListener listener); + Optional multiGet(Consumer multiGetRequestBuilder); - ActionFuture multiGet(MultiGetRequest request); + Optional search(Consumer searchRequestBuilder); - void multiGet(MultiGetRequest request, ActionListener listener); + Stream search(Consumer searchRequestBuilder, + TimeValue scrollTime, int scrollSize); - ActionFuture search(SearchRequest request); - - void search(SearchRequest request, ActionListener listener); + Stream getIds(Consumer queryBuilder); } diff --git a/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java b/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java new file mode 100644 index 0000000..23bfb07 --- /dev/null +++ b/elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java @@ -0,0 +1,7 @@ +package org.xbib.elx.api; + +@FunctionalInterface +public interface SearchClientProvider { + + C getClient(); +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java similarity index 75% rename from elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java rename to elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java index 8ccb489..63923f8 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/AbstractExtendedClient.java +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java @@ -2,10 +2,8 @@ package org.xbib.elx.common; import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -18,14 +16,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; -import org.elasticsearch.action.admin.indices.create.CreateIndexAction; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; -import org.elasticsearch.action.admin.indices.flush.FlushAction; -import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction; import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest; import org.elasticsearch.action.admin.indices.get.GetIndexAction; @@ -34,33 +27,25 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; -import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.health.ClusterHealthStatus; -import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.cluster.metadata.AliasOrIndex; import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.common.xcontent.yaml.YamlXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -69,6 +54,7 @@ import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexAliasAdder; import org.xbib.elx.api.IndexDefinition; @@ -78,13 +64,14 @@ import org.xbib.elx.api.IndexShiftResult; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.MalformedInputException; import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; @@ -97,22 +84,32 @@ import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -public abstract class AbstractExtendedClient implements ExtendedClient { +public abstract class AbstractAdminClient extends AbstractNativeClient implements AdminClient { private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName()); + /** + * The one and only index type name used in the extended client. + * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". + */ + private static final String TYPE_NAME = "doc"; + + /** + * The Elasticsearch client. + */ private ElasticsearchClient client; + private BulkMetric bulkMetric; + private BulkController bulkController; - private final AtomicBoolean closed; + private AtomicBoolean closed; private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() { @Override @@ -148,14 +145,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } }; - protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; - - protected abstract void closeClient() throws IOException; - - protected AbstractExtendedClient() { - closed = new AtomicBoolean(false); - } - @Override public AbstractExtendedClient setClient(ElasticsearchClient client) { this.client = client; @@ -167,6 +156,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return client; } + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + @Override public BulkController getBulkController() { return bulkController; @@ -174,12 +168,16 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public AbstractExtendedClient init(Settings settings) throws IOException { - logger.info("initializing with settings = " + settings.toDelimitedString(',')); + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); if (client == null) { client = createClient(settings); } + if (bulkMetric == null) { + bulkMetric = new DefaultBulkMetric(); + bulkMetric.init(settings); + } if (bulkController == null) { - this.bulkController = new DefaultBulkController(this); + bulkController = new DefaultBulkController(this, bulkMetric); bulkController.init(settings); } return this; @@ -194,8 +192,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public void close() throws IOException { - ensureClient(); + ensureActive(); if (closed.compareAndSet(false, true)) { + if (bulkMetric != null) { + logger.info("closing bulk metric"); + bulkMetric.close(); + } if (bulkController != null) { logger.info("closing bulk controller"); bulkController.close(); @@ -206,7 +208,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String getClusterName() { - ensureClient(); + ensureActive(); try { ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); ClusterStateResponse clusterStateResponse = @@ -226,7 +228,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException { - ensureClient(); + ensureActive(); waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); URL indexSettings = indexDefinition.getSettingsUrl(); if (indexSettings == null) { @@ -261,30 +263,32 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient newIndex(String index) throws IOException { - return newIndex(index, Settings.EMPTY, (Map) null); + return newIndex(index, Settings.EMPTY, (Map) null); } @Override public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException { return newIndex(index, - Settings.settingsBuilder().loadFromStream(".json", settings).build(), - mapping != null ? JsonXContent.jsonXContent.createParser(mapping).mapOrdered() : null); + Settings.builder().loadFromStream(".json", settings, true).build(), + mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null); } @Override public ExtendedClient newIndex(String index, Settings settings) throws IOException { - return newIndex(index, settings, (Map) null); + return newIndex(index, settings, (Map) null); } @Override public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException { return newIndex(index, settings, - mapping != null ? JsonXContent.jsonXContent.createParser(mapping).mapOrdered() : null); + mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null); } @Override - public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) { - ensureClient(); + public ExtendedClient newIndex(String index, Settings settings, Map mapping) throws IOException { + ensureActive(); if (index == null) { logger.warn("no index name given to create index"); return this; @@ -294,35 +298,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { createIndexRequest.settings(settings); } if (mapping != null) { - createIndexRequest.mapping("doc", mapping); + createIndexRequest.mapping(TYPE_NAME, mapping); } CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - if (createIndexResponse.isAcknowledged()) { - return this; - } - throw new IllegalStateException("index creation not acknowledged: " + index); - } - - - @Override - public ExtendedClient newIndex(String index, Settings settings, Map mapping) { - ensureClient(); - if (index == null) { - logger.warn("no index name given to create index"); - return this; - } - CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); - if (settings != null) { - createIndexRequest.settings(settings); - } - if (mapping != null) { - createIndexRequest.mapping("doc", mapping); - } - CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); - if (createIndexResponse.isAcknowledged()) { - return this; - } - throw new IllegalStateException("index creation not acknowledged: " + index); + XContentBuilder builder = XContentFactory.jsonBuilder(); + logger.info("index {} created: {}", index, + Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS))); + return this; } @Override @@ -332,7 +314,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient deleteIndex(String index) { - ensureClient(); + ensureActive(); if (index == null) { logger.warn("no index name given to delete index"); return this; @@ -352,7 +334,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) throws IOException { if (bulkController != null) { - ensureClient(); + ensureActive(); bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); } return this; @@ -361,7 +343,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException { if (bulkController != null) { - ensureClient(); + ensureActive(); bulkController.stopBulkMode(indexDefinition); } return this; @@ -370,7 +352,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { if (bulkController != null) { - ensureClient(); + ensureActive(); bulkController.stopBulkMode(index, timeout, timeUnit); } return this; @@ -378,63 +360,63 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient index(String index, String id, boolean create, String source) { - return index(new IndexRequest().index(index).type("doc").id(id).create(create) - .source(source.getBytes(StandardCharsets.UTF_8))); + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); } @Override public ExtendedClient index(String index, String id, boolean create, BytesReference source) { - return index(new IndexRequest().index(index).type("doc").id(id).create(create) - .source(source)); + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source, XContentType.JSON)); } @Override public ExtendedClient index(IndexRequest indexRequest) { - ensureClient(); - bulkController.bulkIndex(indexRequest); + ensureActive(); + bulkController.index(indexRequest); return this; } @Override public ExtendedClient delete(String index, String id) { - return delete(new DeleteRequest().index(index).type("doc").id(id)); + return delete(new DeleteRequest(index, TYPE_NAME, id)); } @Override public ExtendedClient delete(DeleteRequest deleteRequest) { - ensureClient(); - bulkController.bulkDelete(deleteRequest); + ensureActive(); + bulkController.delete(deleteRequest); return this; } @Override public ExtendedClient update(String index, String id, BytesReference source) { - return update(new UpdateRequest().index(index).type("doc").id(id) - .doc(source)); + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source, XContentType.JSON)); } @Override public ExtendedClient update(String index, String id, String source) { - return update(new UpdateRequest().index(index).type("doc").id(id) - .doc(source.getBytes(StandardCharsets.UTF_8))); + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); } @Override public ExtendedClient update(UpdateRequest updateRequest) { - ensureClient(); - bulkController.bulkUpdate(updateRequest); + ensureActive(); + bulkController.update(updateRequest); return this; } @Override public boolean waitForResponses(long timeout, TimeUnit timeUnit) { - ensureClient(); - return bulkController.waitForBulkResponses(timeout, timeUnit); + ensureActive(); + return bulkController.waitForResponses(timeout, timeUnit); } @Override public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - ensureClient(); + ensureActive(); ensureIndexGiven(index); GetSettingsRequest settingsRequest = new GetSettingsRequest(); settingsRequest.indices(index); @@ -443,13 +425,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (shards > 0) { TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest() - .indices(new String[]{index}) + .indices(index) .waitForActiveShards(shards) .timeout(timeout); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - logger.warn("timeout waiting for recovery"); + logger.error("timeout waiting for recovery"); return false; } } @@ -458,13 +440,15 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { - ensureClient(); + ensureActive(); ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); if (healthResponse != null && healthResponse.isTimedOut()) { - logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); + if (logger.isErrorEnabled()) { + logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name()); + } return false; } return true; @@ -472,7 +456,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { - ensureClient(); + ensureActive(); try { TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, @@ -498,7 +482,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { + public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException { waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations if (level > 0) { updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit); @@ -529,7 +513,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient flushIndex(String index) { if (index != null) { - ensureClient(); + ensureActive(); client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); } return this; @@ -538,7 +522,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public ExtendedClient refreshIndex(String index) { if (index != null) { - ensureClient(); + ensureActive(); client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); } return this; @@ -546,10 +530,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String resolveMostRecentIndex(String alias) { + ensureActive(); if (alias == null) { return null; } - ensureClient(); GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias); GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); @@ -563,6 +547,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return indices.isEmpty() ? alias : indices.iterator().next(); } + @Override public Map getAliases(String index) { if (index == null) { return Collections.emptyMap(); @@ -573,7 +558,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public String resolveAlias(String alias) { - ensureClient(); + ensureActive(); ClusterStateRequest clusterStateRequest = new ClusterStateRequest(); clusterStateRequest.blocks(false); clusterStateRequest.metaData(true); @@ -584,7 +569,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); SortedMap map = clusterStateResponse.getState().getMetaData().getAliasAndIndexLookup(); AliasOrIndex aliasOrIndex = map.get(alias); - return aliasOrIndex != null ? aliasOrIndex.getIndices().iterator().next().getIndex() : null; + return aliasOrIndex != null ? aliasOrIndex.getIndices().iterator().next().getIndex().getName() : null; } @Override @@ -615,7 +600,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexShiftResult shiftIndex(String index, String fullIndexName, List additionalAliases, IndexAliasAdder adder) { - ensureClient(); + ensureActive(); if (index == null) { return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to } @@ -631,8 +616,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { final List moveAliases = new ArrayList<>(); IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest(); if (oldAliasMap == null || !oldAliasMap.containsKey(index)) { - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, index)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName).alias(index)); newAliases.add(index); } // move existing aliases @@ -640,14 +625,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient { for (Map.Entry entry : oldAliasMap.entrySet()) { String alias = entry.getKey(); String filter = entry.getValue(); - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.REMOVE, - oldIndex, alias)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() + .indices(oldIndex).alias(alias)); if (filter != null) { - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, alias).filter(filter)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName).alias(alias).filter(filter)); } else { - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, alias)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName).alias(alias)); } moveAliases.add(alias); } @@ -660,37 +645,31 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (adder != null) { adder.addIndexAlias(indicesAliasesRequest, fullIndexName, additionalAlias); } else { - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, additionalAlias)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName).alias(additionalAlias)); } newAliases.add(additionalAlias); } else { String filter = oldAliasMap.get(additionalAlias); - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.REMOVE, - oldIndex, additionalAlias)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove() + .indices(oldIndex).alias(additionalAlias)); if (filter != null) { - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, additionalAlias).filter(filter)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName).alias(additionalAlias).filter(filter)); } else { - indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - fullIndexName, additionalAlias)); + indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(fullIndexName).alias(additionalAlias)); } moveAliases.add(additionalAlias); } } } if (!indicesAliasesRequest.getAliasActions().isEmpty()) { - StringBuilder sb = new StringBuilder(); - for (IndicesAliasesRequest.AliasActions aliasActions : indicesAliasesRequest.getAliasActions()) { - sb.append("[").append(aliasActions.actionType().name()) - .append(",indices=").append(Arrays.asList(aliasActions.indices())) - .append(",aliases=").append(Arrays.asList(aliasActions.aliases())).append("]"); - } - logger.debug("indices alias request = {}", sb.toString()); + logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString()); IndicesAliasesResponse indicesAliasesResponse = client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet(); - logger.debug("response isAcknowledged = {}", - indicesAliasesResponse.isAcknowledged()); + logger.debug("response isAcknowledged = {} isFragment = {}", + indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment()); } return new SuccessIndexShiftResult(moveAliases, newAliases); } @@ -709,11 +688,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { if (index.equals(fullIndexName)) { return EMPTY_INDEX_PRUNE_RESULT; } - ensureClient(); + ensureActive(); GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE); GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet(); Pattern pattern = Pattern.compile("^(.*?)(\\d+)$"); - logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length); + logger.info("{} indices", getIndexResponse.getIndices().length); List candidateIndices = new ArrayList<>(); for (String s : getIndexResponse.getIndices()) { Matcher m = pattern.matcher(s); @@ -749,30 +728,22 @@ public abstract class AbstractExtendedClient implements ExtendedClient { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest() .indices(indicesToDelete.toArray(s)); DeleteIndexResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet(); - if (response.isAcknowledged()) { - logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s)); - waitForCluster("GREEN", 30L, TimeUnit.SECONDS); - return new SuccessPruneResult(candidateIndices, indicesToDelete, response); - } else { - logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s)); - return new FailPruneResult(candidateIndices, indicesToDelete, response); - } + return new SuccessPruneResult(candidateIndices, indicesToDelete, response); } @Override public Long mostRecentDocument(String index, String timestampfieldname) { - ensureClient(); - SortBuilder sort = SortBuilders - .fieldSort(timestampfieldname) - .order(SortOrder.DESC); - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() - .sort(sort) - .field(timestampfieldname) - .size(1); - SearchRequest searchRequest = new SearchRequest() - .indices(index) - .source(sourceBuilder); - SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); + ensureActive(); + SortBuilder sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.sort(sort); + builder.storedField(timestampfieldname); + builder.size(1); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.source(builder); + SearchResponse searchResponse = + client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); if (searchResponse.getHits().getHits().length == 1) { SearchHit hit = searchResponse.getHits().getHits()[0]; if (hit.getFields().get(timestampfieldname) != null) { @@ -815,7 +786,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException { - boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient)); + boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockAdminClient)); String indexName = settings.get("name", index); String fullIndexName; String dateTimePattern = settings.get("dateTimePattern"); @@ -835,8 +806,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient { .setEnabled(isEnabled) .setIndex(indexName) .setFullIndexName(fullIndexName) - .setSettingsUrl(settings.get("settings")) - .setMappingsUrl(settings.get("mapping")) + .setSettings(findSettingsFrom(settings.get("settings"))) + .setMappings(findMappingsFrom(settings.get("mapping"))) .setDateTimePattern(dateTimePattern) .setIgnoreErrors(settings.getAsBoolean("skiperrors", false)) .setShift(settings.getAsBoolean("shift", true)) @@ -849,29 +820,56 @@ public abstract class AbstractExtendedClient implements ExtendedClient { @Override public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { - ensureClient(); + ensureActive(); if (index == null) { throw new IOException("no index name given"); } - if (key == null) { - throw new IOException("no key given"); + try { + URL url = new URL(string); + try (InputStream inputStream = url.openStream()) { + Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build(); + XContentBuilder builder = JsonXContent.contentBuilder(); + settings.toXContent(builder, ToXContent.EMPTY_PARAMS); + return Strings.toString(builder); + } + } catch (MalformedURLException e) { + return string; } if (value == null) { throw new IOException("no value given"); } - Settings.Builder updateSettingsBuilder = Settings.settingsBuilder(); + Settings.Builder updateSettingsBuilder = Settings.builder(); updateSettingsBuilder.put(key, value.toString()); UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit)); client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); } - private void ensureClient() { + private void ensureActive() { if (this instanceof MockExtendedClient) { return; } - if (client == null) { - throw new IllegalStateException("no client"); + try { + URL url = new URL(string); + try (InputStream inputStream = url.openStream()) { + if (string.endsWith(".json")) { + Map mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered(); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject().map(mappings).endObject(); + return Strings.toString(builder); + } + if (string.endsWith(".yml") || string.endsWith(".yaml")) { + Map mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered(); + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject().map(mappings).endObject(); + return Strings.toString(builder); + } + } + return string; + } catch (MalformedInputException e) { + return string; } } @@ -898,7 +896,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } public void checkMapping(String index) { - ensureClient(); + ensureActive(); GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index); GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); ImmutableOpenMap> map = getMappingsResponse.getMappings(); @@ -914,24 +912,27 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private void checkMapping(String index, String type, MappingMetaData mappingMetaData) { try { - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(QueryBuilders.matchAllQuery()) - .size(0); - SearchRequest searchRequest = new SearchRequest() - .indices(index) - .source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(QueryBuilders.matchAllQuery()); + builder.size(0); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.types(type); + searchRequest.source(builder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); long total = searchResponse.getHits().getTotalHits(); if (total > 0L) { Map fields = new TreeMap<>(); Map root = mappingMetaData.getSourceAsMap(); - checkMapping(index, "", "", root, fields); + checkMapping(index, type, "", "", root, fields); AtomicInteger empty = new AtomicInteger(); Map map = sortByValue(fields); map.forEach((key, value) -> { logger.info("{} {} {}", - key, value, (double) value * 100 / total); + key, + value, + (double) value * 100 / total); if (value == 0) { empty.incrementAndGet(); } @@ -945,7 +946,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @SuppressWarnings("unchecked") - private void checkMapping(String index, + private void checkMapping(String index, String type, String pathDef, String fieldName, Map map, Map fields) { String path = pathDef; @@ -970,17 +971,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient { String fieldType = o instanceof String ? o.toString() : null; // do not recurse into our custom field mapper if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) { - checkMapping(index, path, key, child, fields); + checkMapping(index, type, path, key, child, fields); } } else if ("type".equals(key)) { QueryBuilder filterBuilder = QueryBuilders.existsQuery(path); QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder); - SearchSourceBuilder builder = new SearchSourceBuilder() - .query(queryBuilder) - .size(0); - SearchRequest searchRequest = new SearchRequest() - .indices(index) - .source(builder); + SearchSourceBuilder builder = new SearchSourceBuilder(); + builder.query(queryBuilder); + builder.size(0); + SearchRequest searchRequest = new SearchRequest(); + searchRequest.indices(index); + searchRequest.types(type); + searchRequest.source(builder); SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); fields.put(path, searchResponse.getHits().getTotalHits()); @@ -995,58 +997,38 @@ public abstract class AbstractExtendedClient implements ExtendedClient { return result; } - private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { - switch (timeUnit) { - case DAYS: - return TimeValue.timeValueHours(24 * timeValue); - case HOURS: - return TimeValue.timeValueHours(timeValue); - case MINUTES: - return TimeValue.timeValueMinutes(timeValue); - case SECONDS: - return TimeValue.timeValueSeconds(timeValue); - case MILLISECONDS: - return TimeValue.timeValueMillis(timeValue); - case MICROSECONDS: - return TimeValue.timeValueNanos(1000 * timeValue); - case NANOSECONDS: - return TimeValue.timeValueNanos(timeValue); - default: - throw new IllegalArgumentException("unknown time unit: " + timeUnit); - } - } - private static class SuccessIndexShiftResult implements IndexShiftResult { - List movedAliases; + Collection movedAliases; - List newAliases; + Collection newAliases; - SuccessIndexShiftResult(List movedAliases, List newAliases) { + SuccessIndexShiftResult(Collection movedAliases, Collection newAliases) { this.movedAliases = movedAliases; this.newAliases = newAliases; } @Override - public List getMovedAliases() { + public Collection getMovedAliases() { return movedAliases; } @Override - public List getNewAliases() { + public Collection getNewAliases() { return newAliases; } } private static class SuccessPruneResult implements IndexPruneResult { - List candidateIndices; + Collection candidateIndices; - List indicesToDelete; + Collection indicesToDelete; DeleteIndexResponse response; - SuccessPruneResult(List candidateIndices, List indicesToDelete, + SuccessPruneResult(Collection candidateIndices, + Collection indicesToDelete, DeleteIndexResponse response) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; @@ -1059,48 +1041,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public List getCandidateIndices() { - return candidateIndices; - } - - @Override - public List getDeletedIndices() { - return indicesToDelete; - } - - @Override - public boolean isAcknowledged() { - return response.isAcknowledged(); - } - } - - private static class FailPruneResult implements IndexPruneResult { - - List candidateIndices; - - List indicesToDelete; - - DeleteIndexResponse response; - - FailPruneResult(List candidateIndices, List indicesToDelete, - DeleteIndexResponse response) { - this.candidateIndices = candidateIndices; - this.indicesToDelete = indicesToDelete; - this.response = response; - } - - @Override - public IndexPruneResult.State getState() { - return IndexPruneResult.State.FAIL; - } - - @Override - public List getCandidateIndices() { + public Collection getCandidateIndices() { return candidateIndices; } @Override - public List getDeletedIndices() { + public Collection getDeletedIndices() { return indicesToDelete; } @@ -1112,11 +1058,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient { private static class NothingToDoPruneResult implements IndexPruneResult { - List candidateIndices; + Collection candidateIndices; - List indicesToDelete; + Collection indicesToDelete; - NothingToDoPruneResult(List candidateIndices, List indicesToDelete) { + NothingToDoPruneResult(Collection candidateIndices, List indicesToDelete) { this.candidateIndices = candidateIndices; this.indicesToDelete = indicesToDelete; } @@ -1127,12 +1073,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient { } @Override - public List getCandidateIndices() { + public Collection getCandidateIndices() { return candidateIndices; } @Override - public List getDeletedIndices() { + public Collection getDeletedIndices() { return indicesToDelete; } diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java new file mode 100644 index 0000000..e4c603d --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java @@ -0,0 +1,246 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.admin.indices.create.CreateIndexAction; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.flush.FlushAction; +import org.elasticsearch.action.admin.indices.flush.FlushRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.xbib.elx.api.BulkClient; +import org.xbib.elx.api.BulkController; +import org.xbib.elx.api.BulkMetric; +import org.xbib.elx.api.IndexDefinition; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public abstract class AbstractBulkClient extends AbstractNativeClient implements BulkClient { + + private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName()); + + private BulkMetric bulkMetric; + + private BulkController bulkController; + + @Override + public void init(Settings settings) throws IOException { + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + super.init(settings); + if (bulkMetric == null) { + bulkMetric = new DefaultBulkMetric(); + bulkMetric.init(settings); + } + if (bulkController == null) { + bulkController = new DefaultBulkController(this, bulkMetric); + bulkController.init(settings); + } + } + + @Override + public BulkMetric getBulkMetric() { + return bulkMetric; + } + + @Override + public BulkController getBulkController() { + return bulkController; + } + + @Override + public void flush() throws IOException { + if (bulkController != null) { + bulkController.flush(); + } + } + + @Override + public void close() throws IOException { + ensureClientIsPresent(); + if (closed.compareAndSet(false, true)) { + if (bulkMetric != null) { + logger.info("closing bulk metric"); + bulkMetric.close(); + } + if (bulkController != null) { + logger.info("closing bulk controller"); + bulkController.close(); + } + closeClient(); + } + } + + @Override + public void newIndex(IndexDefinition indexDefinition) throws IOException { + Settings settings = indexDefinition.getSettings() == null ? null : + Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build(); + Map mappings = indexDefinition.getMappings() == null ? null : + JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered(); + newIndex(indexDefinition.getFullIndexName(), settings, mappings); + } + + @Override + public void newIndex(String index) throws IOException { + newIndex(index, Settings.EMPTY, (Map) null); + } + + @Override + public void newIndex(String index, Settings settings) throws IOException { + newIndex(index, settings, (Map) null); + } + + @Override + public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException { + String mappingString = Strings.toString(builder); + Map mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered(); + newIndex(index, settings, mappings); + } + + @Override + public void newIndex(String index, Settings settings, Map mapping) throws IOException { + if (index == null) { + logger.warn("no index name given to create index"); + return; + } + ensureClientIsPresent(); + waitForCluster("YELLOW", 30L, TimeUnit.SECONDS); + CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index); + if (settings != null) { + createIndexRequest.settings(settings); + } + if (mapping != null) { + createIndexRequest.mapping(TYPE_NAME, mapping); + } + CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + logger.info("index {} created: {}", index, + Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS))); + } + + @Override + public void startBulk(IndexDefinition indexDefinition) throws IOException { + startBulk(indexDefinition.getFullIndexName(), -1, 1); + } + + @Override + public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds) + throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds); + } + } + + @Override + public void stopBulk(IndexDefinition indexDefinition) throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.stopBulkMode(indexDefinition); + } + } + + @Override + public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException { + if (bulkController != null) { + ensureClientIsPresent(); + bulkController.stopBulkMode(index, timeout, timeUnit); + } + } + + @Override + public BulkClient index(String index, String id, boolean create, String source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); + } + + @Override + public BulkClient index(String index, String id, boolean create, BytesReference source) { + return index(new IndexRequest(index, TYPE_NAME, id).create(create) + .source(source, XContentType.JSON)); + } + + @Override + public BulkClient index(IndexRequest indexRequest) { + ensureClientIsPresent(); + bulkController.index(indexRequest); + return this; + } + + @Override + public BulkClient delete(String index, String id) { + return delete(new DeleteRequest(index, TYPE_NAME, id)); + } + + @Override + public BulkClient delete(DeleteRequest deleteRequest) { + ensureClientIsPresent(); + bulkController.delete(deleteRequest); + return this; + } + + @Override + public BulkClient update(String index, String id, BytesReference source) { + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source, XContentType.JSON)); + } + + @Override + public BulkClient update(String index, String id, String source) { + return update(new UpdateRequest(index, TYPE_NAME, id) + .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON)); + } + + @Override + public BulkClient update(UpdateRequest updateRequest) { + ensureClientIsPresent(); + bulkController.update(updateRequest); + return this; + } + + @Override + public boolean waitForResponses(long timeout, TimeUnit timeUnit) { + ensureClientIsPresent(); + return bulkController.waitForResponses(timeout, timeUnit); + } + + @Override + public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { + super.updateIndexSetting(index, key, value, timeout, timeUnit); + } + + @Override + public void flushIndex(String index) { + if (index != null) { + ensureClientIsPresent(); + client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet(); + } + } + + @Override + public void refreshIndex(String index) { + if (index != null) { + ensureClientIsPresent(); + client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet(); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java new file mode 100644 index 0000000..5c8c599 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java @@ -0,0 +1,218 @@ +package org.xbib.elx.common; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchTimeoutException; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.xbib.elx.api.NativeClient; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public abstract class AbstractNativeClient implements NativeClient { + + private static final Logger logger = LogManager.getLogger(AbstractNativeClient.class.getName()); + + /** + * The one and only index type name used in the extended client. + * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_". + */ + protected static final String TYPE_NAME = "doc"; + + protected ElasticsearchClient client; + + protected final AtomicBoolean closed; + + protected AbstractNativeClient() { + closed = new AtomicBoolean(false); + } + + @Override + public void setClient(ElasticsearchClient client) { + this.client = client; + } + + @Override + public ElasticsearchClient getClient() { + return client; + } + + protected abstract ElasticsearchClient createClient(Settings settings) throws IOException; + + protected abstract void closeClient() throws IOException; + + + @Override + public void init(Settings settings) throws IOException { + logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(',')); + if (client == null) { + client = createClient(settings); + } + } + + @Override + public String getClusterName() { + ensureClientIsPresent(); + try { + ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear(); + ClusterStateResponse clusterStateResponse = + getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet(); + return clusterStateResponse.getClusterName().value(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString); + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet(); + if (healthResponse != null && healthResponse.isTimedOut()) { + String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name(); + if (logger.isErrorEnabled()) { + logger.error(message); + } + throw new IllegalStateException(message); + } + } + + @Override + public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) { + ensureClientIsPresent(); + try { + TimeValue timeout = toTimeValue(maxWaitTime, timeUnit); + ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE, + new ClusterHealthRequest().timeout(timeout)).actionGet(); + ClusterHealthStatus status = healthResponse.getStatus(); + return status.name(); + } catch (ElasticsearchTimeoutException e) { + logger.warn(e.getMessage(), e); + return "TIMEOUT"; + } catch (NoNodeAvailableException e) { + logger.warn(e.getMessage(), e); + return "DISCONNECTED"; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + return "[" + e.getMessage() + "]"; + } + } + + @Override + public Map getMapping(String index, String mapping) { + GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE) + .setIndices(index) + .setTypes(mapping); + GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet(); + logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap()); + return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap(); + } + + @Override + public long getSearchableDocs(String index) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE) + .setIndices(index) + .setQuery(QueryBuilders.matchAllQuery()) + .setSize(0); + return searchRequestBuilder.execute().actionGet().getHits().getTotalHits(); + } + + @Override + public boolean isIndexExists(String index) { + IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); + indicesExistsRequest.indices(index); + IndicesExistsResponse indicesExistsResponse = + client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); + return indicesExistsResponse.isExists(); + } + + + @Override + public void close() throws IOException { + ensureClientIsPresent(); + if (closed.compareAndSet(false, true)) { + closeClient(); + } + } + + protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException { + ensureClientIsPresent(); + if (index == null) { + throw new IOException("no index name given"); + } + if (key == null) { + throw new IOException("no key given"); + } + if (value == null) { + throw new IOException("no value given"); + } + Settings.Builder updateSettingsBuilder = Settings.builder(); + updateSettingsBuilder.put(key, value.toString()); + UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index) + .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit)); + client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet(); + } + + protected void ensureClientIsPresent() { + if (this instanceof MockAdminClient) { + return; + } + if (client == null) { + throw new IllegalStateException("no client"); + } + } + + protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) { + switch (timeUnit) { + case DAYS: + return TimeValue.timeValueHours(24 * timeValue); + case HOURS: + return TimeValue.timeValueHours(timeValue); + case MINUTES: + return TimeValue.timeValueMinutes(timeValue); + case SECONDS: + return TimeValue.timeValueSeconds(timeValue); + case MILLISECONDS: + return TimeValue.timeValueMillis(timeValue); + case MICROSECONDS: + return TimeValue.timeValueNanos(1000 * timeValue); + case NANOSECONDS: + return TimeValue.timeValueNanos(timeValue); + default: + throw new IllegalArgumentException("unknown time unit: " + timeUnit); + } + } + +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java new file mode 100644 index 0000000..fc37ec9 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java @@ -0,0 +1,158 @@ +package org.xbib.elx.common; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.get.GetAction; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetAction; +import org.elasticsearch.action.get.MultiGetRequestBuilder; +import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.action.search.ClearScrollAction; +import org.elasticsearch.action.search.ClearScrollRequestBuilder; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import org.xbib.elx.api.SearchClient; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Optional; +import java.util.Spliterator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +public abstract class AbstractSearchClient extends AbstractNativeClient implements SearchClient { + + @Override + public Optional get(Consumer getRequestBuilderConsumer) { + GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE); + getRequestBuilderConsumer.accept(getRequestBuilder); + GetResponse getResponse = getRequestBuilder.execute().actionGet(); + return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty(); + } + + @Override + public Optional multiGet(Consumer multiGetRequestBuilderConsumer) { + MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE); + multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder); + MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet(); + return multiGetItemResponse.getResponses().length == 0 ? Optional.empty() : Optional.of(multiGetItemResponse); + } + + @Override + public Optional search(Consumer queryBuilder) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + queryBuilder.accept(searchRequestBuilder); + SearchResponse searchResponse = searchRequestBuilder.execute().actionGet(); + if (searchResponse.getFailedShards() > 0) { + StringBuilder sb = new StringBuilder("Search failed:"); + for (ShardSearchFailure failure : searchResponse.getShardFailures()) { + sb.append("\n").append(failure.reason()); + } + throw new ElasticsearchException(sb.toString()); + } + return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse); + } + + @Override + public Stream search(Consumer queryBuilder, + TimeValue scrollTime, int scrollSize) { + SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE); + queryBuilder.accept(searchRequestBuilder); + searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize); + SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet(); + Stream infiniteResponses = Stream.iterate(originalSearchResponse, + searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE) + .setScrollId(searchResponse.getScrollId()) + .setScroll(scrollTime) + .execute().actionGet()); + Predicate condition = searchResponse -> searchResponse.getHits().getHits().length > 0; + Consumer lastAction = searchResponse -> { + ClearScrollRequestBuilder clearScrollRequestBuilder = + new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE) + .addScrollId(searchResponse.getScrollId()); + clearScrollRequestBuilder.execute().actionGet(); + }; + return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(), + condition, lastAction), false) + .onClose(infiniteResponses::close) + .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits())); + } + + @Override + public Stream getIds(Consumer queryBuilder) { + return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId); + } + + static class TakeWhileSpliterator implements Spliterator { + + private final Spliterator source; + + private final Predicate condition; + + private final Consumer lastElement; + + private final boolean inclusive; + + private final AtomicBoolean checked; + + static TakeWhileSpliterator over(Spliterator source, + Predicate condition, + Consumer lastElement) { + return new TakeWhileSpliterator<>(source, condition, lastElement, true); + } + + private TakeWhileSpliterator(Spliterator source, + Predicate condition, + Consumer lastElement, + boolean inclusive) { + this.source = source; + this.condition = condition; + this.lastElement = lastElement; + this.inclusive = inclusive; + this.checked = new AtomicBoolean(true); + } + + @Override + public boolean tryAdvance(Consumer action) { + return checked.get() && source.tryAdvance(e -> { + if (condition.test(e)) { + action.accept(e); + } else { + if (inclusive && checked.get()) { + action.accept(e); + } + lastElement.accept(e); + checked.set(false); + } + }); + } + + @Override + public Spliterator trySplit() { + return null; + } + + @Override + public long estimateSize() { + return checked.get() ? source.estimateSize() : 0; + } + + @Override + public int characteristics() { + return source.characteristics() &~ Spliterator.SIZED; + } + + @Override + public Comparator getComparator() { + return source.getComparator(); + } + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java index e77a55a..5502e9e 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java +++ b/elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java @@ -8,12 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; -import org.xbib.elx.api.ExtendedClient; -import org.xbib.elx.api.ExtendedClientProvider; +import org.xbib.elx.api.AdminClientProvider; +import org.xbib.elx.api.BulkClientProvider; +import org.xbib.elx.api.NativeClient; +import org.xbib.elx.api.SearchClientProvider; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.ServiceLoader; @SuppressWarnings("rawtypes") @@ -23,11 +23,15 @@ public class ClientBuilder { private final ElasticsearchClient client; + private final ClassLoader classLoader; + private final Settings.Builder settingsBuilder; - private Map, ExtendedClientProvider> providerMap; + private Class adminClientProvider; + + private Class bulkClientProvider; - private Class provider; + private Class searchClientProvider; public ClientBuilder() { this(null); @@ -39,14 +43,9 @@ public class ClientBuilder { public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) { this.client = client; + this.classLoader = classLoader; this.settingsBuilder = Settings.builder(); settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT); - this.providerMap = new HashMap<>(); - ServiceLoader serviceLoader = ServiceLoader.load(ExtendedClientProvider.class, - classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader()); - for (ExtendedClientProvider provider : serviceLoader) { - providerMap.put(provider.getClass(), provider); - } } public static ClientBuilder builder() { @@ -57,8 +56,18 @@ public class ClientBuilder { return new ClientBuilder(client); } - public ClientBuilder provider(Class provider) { - this.provider = provider; + public ClientBuilder setAdminClientProvider(Class adminClientProvider) { + this.adminClientProvider = adminClientProvider; + return this; + } + + public ClientBuilder setBulkClientProvider(Class bulkClientProvider) { + this.bulkClientProvider = bulkClientProvider; + return this; + } + + public ClientBuilder setSearchClientProvider(Class searchClientProvider) { + this.searchClientProvider = searchClientProvider; return this; } @@ -98,14 +107,39 @@ public class ClientBuilder { } @SuppressWarnings("unchecked") - public C build() throws IOException { - if (provider == null) { - throw new IllegalArgumentException("no provider"); - } + public C build() throws IOException { Settings settings = settingsBuilder.build(); logger.log(Level.INFO, "settings = " + settings.toDelimitedString(',')); - return (C) providerMap.get(provider).getExtendedClient() - .setClient(client) - .init(settings); + if (adminClientProvider != null) { + for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(adminClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + if (bulkClientProvider != null) { + for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(bulkClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + if (searchClientProvider != null) { + for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) { + if (provider.getClass().isAssignableFrom(searchClientProvider)) { + C c = (C) provider.getClient(); + c.setClient(client); + c.init(settings); + return c; + } + } + } + throw new IllegalArgumentException("no provider"); } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java index 41c3e8b..67c6984 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java @@ -2,17 +2,19 @@ package org.xbib.elx.common; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; +import org.xbib.elx.api.BulkClient; import org.xbib.elx.api.BulkController; -import org.xbib.elx.api.BulkListener; import org.xbib.elx.api.BulkMetric; import org.xbib.elx.api.BulkProcessor; -import org.xbib.elx.api.ExtendedClient; import org.xbib.elx.api.IndexDefinition; import java.io.IOException; @@ -27,12 +29,10 @@ public class DefaultBulkController implements BulkController { private static final Logger logger = LogManager.getLogger(DefaultBulkController.class); - private final ExtendedClient client; + private final BulkClient client; private final BulkMetric bulkMetric; - private BulkProcessor bulkProcessor; - private final List indexNames; private final Map startBulkRefreshIntervals; @@ -43,11 +43,17 @@ public class DefaultBulkController implements BulkController { private final TimeUnit maxWaitTimeUnit; - private final AtomicBoolean active; + private BulkProcessor bulkProcessor; + + private BulkListener bulkListener; + + private AtomicBoolean active; + + private boolean enableBulkLogging; - public DefaultBulkController(ExtendedClient client) { + public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) { this.client = client; - this.bulkMetric = new DefaultBulkMetric(); + this.bulkMetric = bulkMetric; this.indexNames = new ArrayList<>(); this.active = new AtomicBoolean(false); this.startBulkRefreshIntervals = new HashMap<>(); @@ -56,14 +62,9 @@ public class DefaultBulkController implements BulkController { this.maxWaitTimeUnit = TimeUnit.SECONDS; } - @Override - public BulkMetric getBulkMetric() { - return bulkMetric; - } - @Override public Throwable getLastBulkError() { - return bulkProcessor.getBulkListener().getLastBulkError(); + return bulkListener.getLastBulkError(); } @Override @@ -77,27 +78,22 @@ public class DefaultBulkController implements BulkController { ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(), ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(), "maxVolumePerRequest")); - boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), + this.enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(), Parameters.ENABLE_BULK_LOGGING.getValue()); - BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging); + this.bulkListener = new BulkListener(); this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener) .setBulkActions(maxActionsPerRequest) .setConcurrentRequests(maxConcurrentRequests) .setFlushInterval(flushIngestInterval) .setBulkSize(maxVolumePerRequest) .build(); + this.active.set(true); if (logger.isInfoEnabled()) { logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " + - "flushIngestInterval = {} maxVolumePerRequest = {}, bulk logging = {}", + "flushIngestInterval = {} maxVolumePerRequest = {} bulk logging = {} logger debug = {} from settings = {}", maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest, - enableBulkLogging); + enableBulkLogging, logger.isDebugEnabled(), settings.toDelimitedString(',')); } - this.active.set(true); - } - - @Override - public void inactivate() { - this.active.set(false); } @Override @@ -122,53 +118,65 @@ public class DefaultBulkController implements BulkController { } @Override - public void bulkIndex(IndexRequest indexRequest) { + public void index(IndexRequest indexRequest) { ensureActiveAndBulk(); + if (!active.get()) { + throw new IllegalStateException("inactive"); + } try { - bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id()); + } bulkProcessor.add(indexRequest); } catch (Exception e) { + bulkListener.lastBulkError = e; + active.set(false); if (logger.isErrorEnabled()) { logger.error("bulk add of index failed: " + e.getMessage(), e); } - inactivate(); } } @Override - public void bulkDelete(DeleteRequest deleteRequest) { + public void delete(DeleteRequest deleteRequest) { if (!active.get()) { throw new IllegalStateException("inactive"); } try { - bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id()); + } bulkProcessor.add(deleteRequest); } catch (Exception e) { + bulkListener.lastBulkError = e; + active.set(false); if (logger.isErrorEnabled()) { logger.error("bulk add of delete failed: " + e.getMessage(), e); } - inactivate(); } } @Override - public void bulkUpdate(UpdateRequest updateRequest) { + public void update(UpdateRequest updateRequest) { if (!active.get()) { throw new IllegalStateException("inactive"); } try { - bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id()); + } bulkProcessor.add(updateRequest); } catch (Exception e) { + bulkListener.lastBulkError = e; + active.set(false); if (logger.isErrorEnabled()) { logger.error("bulk add of update failed: " + e.getMessage(), e); } - inactivate(); } } @Override - public boolean waitForBulkResponses(long timeout, TimeUnit timeUnit) { + public boolean waitForResponses(long timeout, TimeUnit timeUnit) { try { return bulkProcessor.awaitFlush(timeout, timeUnit); } catch (InterruptedException e) { @@ -187,7 +195,7 @@ public class DefaultBulkController implements BulkController { @Override public void stopBulkMode(String index, long timeout, TimeUnit timeUnit) throws IOException { flush(); - if (waitForBulkResponses(timeout, timeUnit)) { + if (waitForResponses(timeout, timeUnit)) { if (indexNames.contains(index)) { Long secs = stopBulkRefreshIntervals.get(index); if (secs != null && secs != 0L) { @@ -209,7 +217,6 @@ public class DefaultBulkController implements BulkController { @Override public void close() throws IOException { flush(); - bulkMetric.close(); if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) { for (String index : indexNames) { Long secs = stopBulkRefreshIntervals.get(index); @@ -231,5 +238,92 @@ public class DefaultBulkController implements BulkController { if (bulkProcessor == null) { throw new UnsupportedOperationException("bulk processor not present"); } + if (bulkListener == null) { + throw new UnsupportedOperationException("bulk listener not present"); + } + } + + private class BulkListener implements DefaultBulkProcessor.Listener { + + private final Logger logger = LogManager.getLogger(BulkListener.class.getName()); + + private Throwable lastBulkError = null; + + @Override + public void beforeBulk(long executionId, BulkRequest request) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().inc(); + int n = request.numberOfActions(); + bulkMetric.getSubmitted().inc(n); + bulkMetric.getCurrentIngestNumDocs().inc(n); + bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes()); + } + if (enableBulkLogging && logger.isDebugEnabled()) { + logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]", + executionId, + request.numberOfActions(), + request.estimatedSizeInBytes(), + l); + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + long l = 0; + if (bulkMetric != null) { + l = bulkMetric.getCurrentIngest().getCount(); + bulkMetric.getCurrentIngest().dec(); + bulkMetric.getSucceeded().inc(response.getItems().length); + } + int n = 0; + for (BulkItemResponse itemResponse : response.getItems()) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId()); + } + if (itemResponse.isFailed()) { + n++; + if (bulkMetric != null) { + bulkMetric.getSucceeded().dec(1); + bulkMetric.getFailed().inc(1); + } + } + } + if (enableBulkLogging && logger.isDebugEnabled() && bulkMetric != null) { + logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests", + executionId, + bulkMetric.getSucceeded().getCount(), + bulkMetric.getFailed().getCount(), + response.getTook().millis(), + l); + } + if (n > 0) { + if (enableBulkLogging && logger.isErrorEnabled()) { + logger.error("bulk [{}] failed with {} failed items, failure message = {}", + executionId, n, response.buildFailureMessage()); + } + } else { + if (bulkMetric != null) { + bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length); + } + } + } + + @Override + public void afterBulk(long executionId, BulkRequest request, Throwable failure) { + if (bulkMetric != null) { + bulkMetric.getCurrentIngest().dec(); + } + lastBulkError = failure; + active.set(false); + if (enableBulkLogging && logger.isErrorEnabled()) { + logger.error("after bulk [" + executionId + "] error", failure); + } + } + + Throwable getLastBulkError() { + return lastBulkError; + } } } diff --git a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java index 52127e1..8d2d3fd 100644 --- a/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java +++ b/elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java @@ -3,8 +3,6 @@ package org.xbib.elx.common; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.api.IndexRetention; -import java.net.MalformedURLException; -import java.net.URL; import java.util.concurrent.TimeUnit; public class DefaultIndexDefinition implements IndexDefinition { @@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition { private String dateTimePattern; - private URL settingsUrl; + private String settings; - private URL mappingsUrl; + private String mappings; private boolean enabled; @@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition { } @Override - public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException { - this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null; + public IndexDefinition setSettings(String settings) { + this.settings = settings; return this; } @Override - public IndexDefinition setSettingsUrl(URL settingsUrl) { - this.settingsUrl = settingsUrl; - return this; - } - - @Override - public URL getSettingsUrl() { - return settingsUrl; - } - - @Override - public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException { - this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null; - return this; + public String getSettings() { + return settings; } @Override - public IndexDefinition setMappingsUrl(URL mappingsUrl) { - this.mappingsUrl = mappingsUrl; + public IndexDefinition setMappings(String mappings) { + this.mappings = mappings; return this; } @Override - public URL getMappingsUrl() { - return mappingsUrl; + public String getMappings() { + return mappings; } @Override diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java new file mode 100644 index 0000000..a63c989 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java @@ -0,0 +1,59 @@ +package org.xbib.elx.common; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockAdminClient extends AbstractAdminClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient() { + } + + @Override + public MockAdminClient deleteIndex(String index) { + return this; + } + + @Override + public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { + } + + @Override + public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { + return this; + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java new file mode 100644 index 0000000..ca1917b --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.AdminClientProvider; + +public class MockAdminClientProvider implements AdminClientProvider { + @Override + public MockAdminClient getClient() { + return new MockAdminClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java new file mode 100644 index 0000000..7c78f83 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java @@ -0,0 +1,99 @@ +package org.xbib.elx.common; + +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import java.util.concurrent.TimeUnit; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockBulkClient extends AbstractBulkClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + public String getClusterName() { + return null; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient() { + } + + @Override + public MockBulkClient index(String index, String id, boolean create, String source) { + return this; + } + + @Override + public MockBulkClient delete(String index, String id) { + return this; + } + + @Override + public MockBulkClient update(String index, String id, String source) { + return this; + } + + @Override + public MockBulkClient index(IndexRequest indexRequest) { + return this; + } + + @Override + public MockBulkClient delete(DeleteRequest deleteRequest) { + return this; + } + + @Override + public MockBulkClient update(UpdateRequest updateRequest) { + return this; + } + + @Override + public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { + } + + @Override + public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { + } + + @Override + public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { + return true; + } + + @Override + public void refreshIndex(String index) { + } + + @Override + public void flushIndex(String index) { + } + + + @Override + public void flush() { + // nothing to do + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java new file mode 100644 index 0000000..93e032d --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.BulkClientProvider; + +public class MockBulkClientProvider implements BulkClientProvider { + + @Override + public MockBulkClient getClient() { + return new MockBulkClient(); + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java deleted file mode 100644 index 647894b..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.xbib.elx.common; - -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.settings.Settings; - -import java.util.concurrent.TimeUnit; - -/** - * A mocked client, it does not perform any actions on a cluster. Useful for testing. - */ -public class MockExtendedClient extends AbstractExtendedClient { - - @Override - public ElasticsearchClient getClient() { - return null; - } - - @Override - public MockExtendedClient init(Settings settings) { - return this; - } - - @Override - protected ElasticsearchClient createClient(Settings settings) { - return null; - } - - @Override - protected void closeClient() { - } - - @Override - public MockExtendedClient index(String index, String id, boolean create, String source) { - return this; - } - - @Override - public MockExtendedClient delete(String index, String id) { - return this; - } - - @Override - public MockExtendedClient update(String index, String id, String source) { - return this; - } - - @Override - public MockExtendedClient index(IndexRequest indexRequest) { - return this; - } - - @Override - public MockExtendedClient delete(DeleteRequest deleteRequest) { - return this; - } - - @Override - public MockExtendedClient update(UpdateRequest updateRequest) { - return this; - } - - @Override - public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) { - return this; - } - - @Override - public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) { - return this; - } - - @Override - public MockExtendedClient newIndex(String index) { - return this; - } - - @Override - public MockExtendedClient deleteIndex(String index) { - return this; - } - - @Override - public MockExtendedClient refreshIndex(String index) { - return this; - } - - @Override - public MockExtendedClient flushIndex(String index) { - return this; - } - - @Override - public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) { - return true; - } - - @Override - public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) { - return this; - } - - @Override - public void flush() { - // nothing to do - } - - @Override - public void close() { - // nothing to do - } -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java deleted file mode 100644 index 87e65cc..0000000 --- a/elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.common; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class MockExtendedClientProvider implements ExtendedClientProvider { - @Override - public MockExtendedClient getExtendedClient() { - return new MockExtendedClient(); - } -} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java new file mode 100644 index 0000000..07c0091 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java @@ -0,0 +1,38 @@ +package org.xbib.elx.common; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; + +/** + * A mocked client, it does not perform any actions on a cluster. Useful for testing. + */ +public class MockSearchClient extends AbstractSearchClient { + + @Override + public ElasticsearchClient getClient() { + return null; + } + + @Override + public void init(Settings settings) { + } + + @Override + public String getClusterName() { + return null; + } + + @Override + protected ElasticsearchClient createClient(Settings settings) { + return null; + } + + @Override + protected void closeClient() { + } + + @Override + public void close() { + // nothing to do + } +} diff --git a/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java new file mode 100644 index 0000000..c943fb4 --- /dev/null +++ b/elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.common; + +import org.xbib.elx.api.SearchClientProvider; + +public class MockSearchClientProvider implements SearchClientProvider { + + @Override + public MockSearchClient getClient() { + return new MockSearchClient(); + } +} diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..1a2c887 --- /dev/null +++ b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.common.MockAdminClientProvider \ No newline at end of file diff --git a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 9729b83..0000000 --- a/elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.common.MockExtendedClientProvider \ No newline at end of file diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java new file mode 100644 index 0000000..c7123e8 --- /dev/null +++ b/elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java @@ -0,0 +1,21 @@ +package org.xbib.elx.common.test; + +import org.junit.jupiter.api.Test; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.MockAdminClient; +import org.xbib.elx.common.MockAdminClientProvider; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class MockAdminClientProviderTest { + + @Test + void testMockAdminProvider() throws IOException { + MockAdminClient client = ClientBuilder.builder() + .setAdminClientProvider(MockAdminClientProvider.class) + .build(); + assertNotNull(client); + } +} diff --git a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java b/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java deleted file mode 100644 index e21817c..0000000 --- a/elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.xbib.elx.common.test; - -import org.junit.jupiter.api.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.MockExtendedClient; -import org.xbib.elx.common.MockExtendedClientProvider; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertNotNull; - -class MockExtendedClientProviderTest { - - @Test - void testMockExtendedProvider() throws IOException { - MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build(); - assertNotNull(client); - } -} diff --git a/elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java b/elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java new file mode 100644 index 0000000..3c06622 --- /dev/null +++ b/elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java @@ -0,0 +1,155 @@ +package org.elasticsearch.action.admin.indices.get; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import org.apache.lucene.util.CollectionUtil; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.MappingMetaData; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.xbib.elx.http.HttpAction; +import org.xbib.netty.http.client.api.Request; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class HttpGetIndexAction extends HttpAction { + + @Override + public GetIndexAction getActionInstance() { + return GetIndexAction.INSTANCE; + } + + @Override + protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException { + List list = getIndexRequest.indices().length == 0 ? + List.of("*") : Arrays.asList(getIndexRequest.indices()); + String command = "/" + String.join(",", list); + logger.info("command = " + command); + return newGetRequest(url, command); + } + + @Override + protected CheckedFunction entityParser() { + return this::fromXContent; + } + + @Override + protected GetIndexResponse emptyResponse() { + return new GetIndexResponse(); + } + + private GetIndexResponse fromXContent(XContentParser parser) throws IOException { + ImmutableOpenMap.Builder> aliases = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder> mappings = ImmutableOpenMap.builder(); + ImmutableOpenMap.Builder settings = ImmutableOpenMap.builder(); + List indices = new ArrayList<>(); + if (parser.currentToken() == null) { + parser.nextToken(); + } + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + while (!parser.isClosed()) { + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + String indexName = parser.currentName(); + indices.add(indexName); + IndexEntry indexEntry = parseIndexEntry(parser); + CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias)); + aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases)); + mappings.put(indexName, indexEntry.indexMappings); + settings.put(indexName, indexEntry.indexSettings); + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } else { + parser.nextToken(); + } + } + return new GetIndexResponse(indices.toArray(new String[0]), + mappings.build(), + aliases.build(), + settings.build()); + } + + private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException { + List indexAliases = null; + ImmutableOpenMap indexMappings = null; + Settings indexSettings = null; + Settings indexDefaultSettings = null; + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + switch (parser.currentName()) { + case "aliases": + indexAliases = parseAliases(parser); + break; + case "mappings": + indexMappings = parseMappings(parser); + break; + case "settings": + indexSettings = Settings.fromXContent(parser); + break; + case "defaults": + indexDefaultSettings = Settings.fromXContent(parser); + break; + default: + parser.skipChildren(); + } + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + } + return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings); + } + + private static List parseAliases(XContentParser parser) throws IOException { + List indexAliases = new ArrayList<>(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + indexAliases.add(AliasMetaData.Builder.fromXContent(parser)); + } + return indexAliases; + } + + private static ImmutableOpenMap parseMappings(XContentParser parser) throws IOException { + ImmutableOpenMap.Builder indexMappings = ImmutableOpenMap.builder(); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + parser.nextToken(); + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + String mappingType = parser.currentName(); + indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map())); + } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { + parser.skipChildren(); + } + } + return indexMappings.build(); + } + + private static class IndexEntry { + List indexAliases = new ArrayList<>(); + ImmutableOpenMap indexMappings = ImmutableOpenMap.of(); + Settings indexSettings = Settings.EMPTY; + Settings indexDefaultSettings = Settings.EMPTY; + + IndexEntry(List indexAliases, ImmutableOpenMap indexMappings, + Settings indexSettings, Settings indexDefaultSettings) { + if (indexAliases != null) { + this.indexAliases = indexAliases; + } + if (indexMappings != null) { + this.indexMappings = indexMappings; + } + if (indexSettings != null) { + this.indexSettings = indexSettings; + } + if (indexDefaultSettings != null) { + this.indexDefaultSettings = indexDefaultSettings; + } + } + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java deleted file mode 100644 index e91c923..0000000 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.http; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedHttpClientProvider implements ExtendedClientProvider { - @Override - public ExtendedHttpClient getExtendedClient() { - return new ExtendedHttpClient(); - } -} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java index 1b6399b..485bd15 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAction.java @@ -154,15 +154,35 @@ public abstract class HttpAction httpActionContext) { + try (XContentParser parser = XContentFactory.xContent(XContentType.JSON) + .createParser(httpActionContext.getExtendedHttpClient().getRegistry(), + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) { + return errorParser().apply(parser); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return new ElasticsearchStatusException(e.getMessage(), RestStatus.INTERNAL_SERVER_ERROR, e); + } + } + + protected CheckedFunction errorParser() { + return BytesRestResponse::errorFromXContent; + } + + protected abstract Request.Builder createHttpRequest(String baseUrl, R request) throws IOException; protected abstract CheckedFunction entityParser(); diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java index 0a0abeb..78f5b16 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java @@ -13,7 +13,7 @@ import org.xbib.netty.http.client.transport.Transport; */ public class HttpActionContext { - private final ExtendedHttpClient extendedHttpClient; + private final HttpClientHelper helper; private final R request; @@ -23,14 +23,14 @@ public class HttpActionContext> ActionFuture execute(Action action, Request request) { + return helper.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public > RequestBuilder prepareExecute(Action action) { + return helper.prepareExecute(action); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java new file mode 100644 index 0000000..4b606aa --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java @@ -0,0 +1,10 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.AdminClientProvider; + +public class HttpAdminClientProvider implements AdminClientProvider { + @Override + public HttpAdminClient getClient() { + return new HttpAdminClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java new file mode 100644 index 0000000..369853b --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java @@ -0,0 +1,61 @@ +package org.xbib.elx.http; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +/** + * Elasticsearch HTTP bulk client. + */ +public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient { + + private final HttpClientHelper helper; + + public HttpBulkClient() { + this.helper = new HttpClientHelper(); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init(settings); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() throws IOException { + helper.closeClient(); + } + + @Override + public > ActionFuture execute(Action action, Request request) { + return helper.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public > RequestBuilder prepareExecute(Action action) { + return helper.prepareExecute(action); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java new file mode 100644 index 0000000..cbf1945 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.BulkClientProvider; + +public class HttpBulkClientProvider implements BulkClientProvider { + + @Override + public HttpBulkClient getClient() { + return new HttpBulkClient(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java similarity index 64% rename from elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java rename to elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java index 1f51857..a3ed0f9 100644 --- a/elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClient.java +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java @@ -1,5 +1,6 @@ package org.xbib.elx.http; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.Action; @@ -12,52 +13,79 @@ import org.elasticsearch.action.GenericAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.threadpool.ThreadPool; -import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.elx.common.AbstractAdminClient; +import org.xbib.net.URL; import org.xbib.netty.http.client.Client; - import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.ServiceLoader; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Elasticsearch HTTP client. */ -public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient { +public class HttpClientHelper extends AbstractAdminClient implements ElasticsearchClient { - private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class); + private static final Logger logger = LogManager.getLogger(HttpClientHelper.class); private Client nettyHttpClient; private final ClassLoader classLoader; + private final NamedXContentRegistry registry; + @SuppressWarnings("rawtypes") private final Map actionMap; private String url; - public ExtendedHttpClient() { - this.classLoader = ExtendedHttpClient.class.getClassLoader(); + public ExtendedHttpClient(List namedXContentEntries, ClassLoader classLoader) { + this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(), + namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList())); + this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader(); this.actionMap = new HashMap<>(); } @Override @SuppressWarnings({"unchecked", "rawtypes"}) - public ExtendedHttpClient init(Settings settings) throws IOException { + public void init(Settings settings) throws IOException { super.init(settings); - if (settings == null) { - return null; + if (settings.hasValue("url")) { + this.url = settings.get("url"); + } else if (settings.hasValue("host")) { + this.url = URL.http() + .host(settings.get("host")).port(settings.getAsInt("port", 9200)) + .build() + .toExternalForm(); } - this.url = settings.get("url"); ServiceLoader httpActionServiceLoader = ServiceLoader.load(HttpAction.class, classLoader); for (HttpAction httpAction : httpActionServiceLoader) { httpAction.setSettings(settings); actionMap.put(httpAction.getActionInstance(), httpAction); } - this.nettyHttpClient = Client.builder().enableDebug().build(); - logger.info("extended HTTP client initialized with {} actions", actionMap.size()); - return this; + Client.Builder clientBuilder = Client.builder(); + if (settings.hasValue("debug")) { + clientBuilder.enableDebug(); + } + this.nettyHttpClient = clientBuilder.build(); + logger.log(Level.DEBUG, "extended HTTP client initialized, settings = {}, url = {}, {} actions", + settings, url, actionMap.size()); + } + + private static List getNamedXContents() { + return new ArrayList<>(); + } + + public NamedXContentRegistry getRegistry() { + return registry; } public Client internalClient() { @@ -104,7 +132,8 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti @Override public ThreadPool threadPool() { - throw new UnsupportedOperationException(); + logger.log(Level.DEBUG, "returning null for threadPool() request"); + return null; } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -116,9 +145,7 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti } try { HttpActionContext httpActionContext = new HttpActionContext(this, request, url); - if (logger.isDebugEnabled()) { - logger.debug("submitting request {} to URL {}", request, url); - } + logger.log(Level.DEBUG, "url = " + url); httpAction.execute(httpActionContext, listener); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java new file mode 100644 index 0000000..07e9249 --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java @@ -0,0 +1,61 @@ +package org.xbib.elx.http; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +/** + * Elasticsearch HTTP search client. + */ +public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient { + + private final HttpClientHelper helper; + + public HttpSearchClient() { + this.helper = new HttpClientHelper(); + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init(settings); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() throws IOException { + helper.closeClient(); + } + + @Override + public > ActionFuture execute(Action action, Request request) { + return helper.execute(action, request); + } + + @Override + public > void execute(Action action, Request request, ActionListener listener) { + helper.execute(action, request, listener); + } + + @Override + public > RequestBuilder prepareExecute(Action action) { + return helper.prepareExecute(action); + } + + @Override + public ThreadPool threadPool() { + return helper.threadPool(); + } +} diff --git a/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java new file mode 100644 index 0000000..9103c1a --- /dev/null +++ b/elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.http; + +import org.xbib.elx.api.SearchClientProvider; + +public class HttpSearchClientProvider implements SearchClientProvider { + + @Override + public HttpSearchClient getClient() { + return new HttpSearchClient(); + } +} diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..f09a6e4 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpAdminClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..39fbdef --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpBulkClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 0c75f14..0000000 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.http.ExtendedHttpClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..882ed38 --- /dev/null +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.http.HttpSearchClientProvider \ No newline at end of file diff --git a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction index 3d3ea95..c42126b 100644 --- a/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction +++ b/elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction @@ -1,3 +1,18 @@ +org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction +org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction +org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction +org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction +org.xbib.elx.http.action.admin.cluster.state.HttpClusterStateAction +org.xbib.elx.http.action.admin.indices.alias.HttpIndicesAliasesAction +org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction +org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction +org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction +org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction +org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction +org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction +org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction +org.xbib.elx.http.action.bulk.HttpBulkAction +org.xbib.elx.http.action.index.HttpIndexAction org.xbib.elx.http.action.search.HttpSearchAction org.xbib.elx.http.action.get.HttpGetAction org.xbib.elx.http.action.get.HttpMultiGetAction diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java similarity index 62% rename from elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java rename to elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java index 3733275..b0029f1 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java +++ b/elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java @@ -1,4 +1,4 @@ -package org.xbib.elx.node.test; +package org.xbib.elx.http.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -8,21 +8,21 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.http.HttpBulkClient; +import org.xbib.elx.http.HttpBulkClientProvider; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -32,11 +32,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(TestExtension.class) class ClientTest { - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); + private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName()); - private static final Long ACTIONS = 1000L; + private static final Long ACTIONS = 100L; - private static final Long MAX_ACTIONS_PER_REQUEST = 100L; + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; private final TestExtension.Helper helper; @@ -44,20 +44,45 @@ class ClientTest { this.helper = helper; } + @Test + void testSingleDoc() throws Exception { + final ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + client.newIndex("test"); + client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + client.flush(); + client.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, client.getBulkMetric().getSucceeded().getCount()); + if (client.getBulkController().getLastBulkError() != null) { + logger.error("error", client.getBulkController().getLastBulkError()); + } + assertNull(client.getBulkController().getLastBulkError()); + client.close(); + } + } + @Test void testNewIndex() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); - client.newIndex("test1"); + client.newIndex("test"); client.close(); } @Test void testMapping() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) .build(); XContentBuilder builder = JsonXContent.contentBuilder() @@ -70,68 +95,44 @@ class ClientTest { .endObject() .endObject() .endObject(); - client.newIndex("test2", Settings.EMPTY, builder.string()); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2"); + client.newIndex("test", Settings.EMPTY, Strings.toString(builder)); + GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test"); GetMappingsResponse getMappingsResponse = client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc")); + assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc")); client.close(); } - @Test - void testSingleDoc() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) - .build(); - try { - client.newIndex("test3"); - client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - @Test void testRandomDocs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test4"); + client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test4", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); - client.waitForResponses(60L, TimeUnit.SECONDS); + client.waitForResponses(30L, TimeUnit.SECONDS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); + } catch (Exception e) { + logger.error(e.getMessage(), e); } finally { - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test4"); + client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test4") - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0); + .setQuery(QueryBuilders.matchAllQuery()).setSize(0); assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.close(); @@ -141,55 +142,60 @@ class ClientTest { @Test void testThreadedRandomDocs() throws Exception { int maxthreads = Runtime.getRuntime().availableProcessors(); - long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; - final long actions = ACTIONS; - logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) - .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final Long actions = ACTIONS; + logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final ExtendedHttpClient client = ClientBuilder.builder() + .provider(ExtendedHttpClientProvider.class) + .put(helper.getHttpSettings()) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) .build(); try { - client.newIndex("test5") - .startBulk("test5", -1, 1000); - ThreadPoolExecutor pool = EsExecutors.newFixed("nodeclient-test", maxthreads, 30, - EsExecutors.daemonThreadFactory("nodeclient-test")); + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + client.newIndex("test", settings, (String)null) + .startBulk("test", 0, 1000); + logger.info("index created"); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); final CountDownLatch latch = new CountDownLatch(maxthreads); for (int i = 0; i < maxthreads; i++) { - pool.execute(() -> { + executorService.execute(() -> { for (int i1 = 0; i1 < actions; i1++) { - client.index("test5", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}"); } latch.countDown(); }); } logger.info("waiting for latch..."); - if (latch.await(60, TimeUnit.SECONDS)) { + if (latch.await(60L, TimeUnit.SECONDS)) { logger.info("flush..."); client.flush(); client.waitForResponses(60L, TimeUnit.SECONDS); - logger.info("pool shutdown..."); - pool.shutdown(); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(60L, TimeUnit.SECONDS); logger.info("pool is shut down"); } else { logger.warn("latch timeout"); } + client.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); + } catch (Exception e) { + logger.error(e.getMessage(), e); } finally { - client.stopBulk("test5", 60L, TimeUnit.SECONDS); - assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test5"); + client.refreshIndex("test"); SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test5") - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0); + .setQuery(QueryBuilders.matchAllQuery()).setSize(0); assertEquals(maxthreads * actions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); client.close(); diff --git a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java b/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java deleted file mode 100644 index 36f038a..0000000 --- a/elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java +++ /dev/null @@ -1,122 +0,0 @@ -package org.xbib.elx.http.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.refresh.RefreshAction; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.get.GetAction; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetAction; -import org.elasticsearch.action.get.MultiGetRequest; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.index.IndexAction; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.junit.Ignore; -import org.junit.Test; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.http.ExtendedHttpClient; -import org.xbib.elx.http.ExtendedHttpClientProvider; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class ClientTest extends TestBase { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - - @Ignore - @Test - public void testGet() throws Exception { - try (ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put("url", "http://" + host + ":" + httpPort) - .build()) { - IndexRequest indexRequest = new IndexRequest(); - indexRequest.index("test"); - indexRequest.type("test"); - indexRequest.id("1"); - indexRequest.source("test", "Hello Jörg"); - IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); - client("1").execute(RefreshAction.INSTANCE, new RefreshRequest()); - - GetRequest getRequest = new GetRequest(); - getRequest.index("test"); - getRequest.type("test"); - getRequest.id("1"); - - GetResponse getResponse = client.execute(GetAction.INSTANCE, getRequest).actionGet(); - - assertTrue(getResponse.isExists()); - assertEquals("{\"test\":\"Hello Jörg\"}", getResponse.getSourceAsString()); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } - } - - @Ignore - @Test - public void testMultiGet() throws Exception { - try (ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put("url", "http://" + host + ":" + httpPort) - .build()) { - IndexRequest indexRequest = new IndexRequest(); - indexRequest.index("test"); - indexRequest.type("test"); - indexRequest.id("1"); - indexRequest.source("test", "Hello Jörg"); - IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); - client("1").execute(RefreshAction.INSTANCE, new RefreshRequest()); - - MultiGetRequest multiGetRequest = new MultiGetRequest(); - multiGetRequest.add("test", "test", "1"); - - MultiGetResponse multiGetResponse = client.execute(MultiGetAction.INSTANCE, multiGetRequest).actionGet(); - - assertEquals(1, multiGetResponse.getResponses().length); - assertEquals("{\"test\":\"Hello Jörg\"}", multiGetResponse.getResponses()[0].getResponse().getSourceAsString()); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } - } - - @Test - public void testSearchDoc() throws Exception { - try (ExtendedHttpClient client = ClientBuilder.builder() - .provider(ExtendedHttpClientProvider.class) - .put("url", "http://" + host + ":" + httpPort) - .build()) { - IndexRequest indexRequest = new IndexRequest(); - indexRequest.index("test"); - indexRequest.type("test"); - indexRequest.id("1"); - indexRequest.source("test", "Hello Jörg"); - IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet(); - client("1").execute(RefreshAction.INSTANCE, new RefreshRequest()); - - SearchSourceBuilder builder = new SearchSourceBuilder(); - builder.query(QueryBuilders.matchAllQuery()); - SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test"); - searchRequest.types("test"); - searchRequest.source(builder); - SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); - long hits = searchResponse.getHits().getTotalHits(); - assertEquals(1, hits); - logger.info("hits = {} source = {}", hits, searchResponse.getHits().getHits()[0].getSourceAsString()); - assertEquals("{\"test\":\"Hello Jörg\"}", searchResponse.getHits().getHits()[0].getSourceAsString()); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } - } -} diff --git a/elx-http/src/test/resources/log4j2.xml b/elx-http/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-http/src/test/resources/log4j2.xml rename to elx-http/src/test/resources/log4j2-test.xml diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java deleted file mode 100644 index 46a4e9a..0000000 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.xbib.elx.node; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedNodeClientProvider implements ExtendedClientProvider { - @Override - public ExtendedNodeClient getExtendedClient() { - return new ExtendedNodeClient(); - } -} diff --git a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java similarity index 90% rename from elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java rename to elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java index 217fbef..e1715f9 100644 --- a/elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java @@ -8,15 +8,15 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; import org.elasticsearch.node.Node; import org.elasticsearch.plugins.Plugin; -import org.xbib.elx.common.AbstractExtendedClient; +import org.xbib.elx.common.AbstractAdminClient; import java.io.IOException; import java.util.Collection; import java.util.Collections; -public class ExtendedNodeClient extends AbstractExtendedClient { +public class NodeAdminClient extends AbstractAdminClient { - private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName()); + private static final Logger logger = LogManager.getLogger(NodeAdminClient.class.getName()); private Node node; diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java new file mode 100644 index 0000000..ec949e0 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.AdminClientProvider; + +public class NodeAdminClientProvider implements AdminClientProvider { + + @Override + public NodeAdminClient getClient() { + return new NodeAdminClient(); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java new file mode 100644 index 0000000..d469609 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java @@ -0,0 +1,68 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class NodeBulkClient extends AbstractBulkClient { + + private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName()); + + private Node node; + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings == null) { + return null; + } + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); + logger.info("creating node client on {} with effective settings {}", + version, Strings.toString(builder)); + Collection> plugins = Collections.emptyList(); + this.node = new BulkNode(new Environment(effectiveSettings, null), plugins); + try { + node.start(); + } catch (Exception e) { + throw new IOException(e); + } + return node.client(); + } + + @Override + protected void closeClient() throws IOException { + if (node != null) { + logger.debug("closing node client"); + node.close(); + } + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, classpathPlugins); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java new file mode 100644 index 0000000..f40a021 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.BulkClientProvider; + +public class NodeBulkClientProvider implements BulkClientProvider { + + @Override + public NodeBulkClient getClient() { + return new NodeBulkClient(); + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java new file mode 100644 index 0000000..ab7f35d --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java @@ -0,0 +1,68 @@ +package org.xbib.elx.node; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.env.Environment; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +public class NodeSearchClient extends AbstractSearchClient { + + private static final Logger logger = LogManager.getLogger(NodeSearchClient.class.getName()); + + private Node node; + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings == null) { + return null; + } + String version = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.runtime.version") + + " " + System.getProperty("java.vm.version"); + Settings effectiveSettings = Settings.builder().put(settings) + .put("node.client", true) + .put("node.master", false) + .put("node.data", false) + .build(); + XContentBuilder builder = XContentFactory.jsonBuilder(); + effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true"))); + logger.info("creating node client on {} with effective settings {}", + version, Strings.toString(builder)); + Collection> plugins = Collections.emptyList(); + this.node = new BulkNode(new Environment(effectiveSettings, null), plugins); + try { + node.start(); + } catch (Exception e) { + throw new IOException(e); + } + return node.client(); + } + + @Override + protected void closeClient() throws IOException { + if (node != null) { + logger.debug("closing node client"); + node.close(); + } + } + + private static class BulkNode extends Node { + + BulkNode(Environment env, Collection> classpathPlugins) { + super(env, classpathPlugins); + } + } +} diff --git a/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java new file mode 100644 index 0000000..233c2a4 --- /dev/null +++ b/elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.node; + +import org.xbib.elx.api.SearchClientProvider; + +public class NodeSearchClientProvider implements SearchClientProvider { + + @Override + public NodeSearchClient getClient() { + return new NodeSearchClient(); + } +} diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..acc3b0c --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeAdminClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..62630f5 --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeBulkClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 372aaad..0000000 --- a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.node.ExtendedNodeClientProvider \ No newline at end of file diff --git a/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..54b437c --- /dev/null +++ b/elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.node.NodeSearchClientProvider \ No newline at end of file diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java new file mode 100644 index 0000000..16b2c69 --- /dev/null +++ b/elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java @@ -0,0 +1,174 @@ +package org.xbib.elx.node.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName()); + + private static final Long ACTIONS = 10000L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + bulkClient.newIndex("test"); + bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + bulkClient.newIndex("test"); + bulkClient.close(); + } + + @Test + void testMapping() throws Exception { + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .build(); + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties")); + bulkClient.close(); + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final long actions = ACTIONS; + logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(30L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(30L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java index f2a1689..f7e35a2 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java @@ -2,17 +2,12 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.search.builder.SearchSourceBuilder; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -25,9 +20,9 @@ class DuplicateIDTest { private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private static final Long MAX_ACTIONS_PER_REQUEST = 10L; + private static final Long ACTIONS = 100L; - private static final Long ACTIONS = 50L; + private static final Long MAX_ACTIONS_PER_REQUEST = 5L; private final TestExtension.Helper helper; @@ -38,23 +33,24 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) .build(); try { - client.newIndex("test_dup"); + client.newIndex("test"); for (int i = 0; i < ACTIONS; i++) { - client.index("test_dup", helper.randomString(1), false, + client.index("test", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - client.refreshIndex("test_dup"); + client.refreshIndex("test"); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); SearchRequest searchRequest = new SearchRequest(); - searchRequest.indices("test_dup"); + searchRequest.indices("test"); + searchRequest.types("test"); searchRequest.source(builder); long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); logger.info("hits = {}", hits); @@ -63,11 +59,11 @@ class DuplicateIDTest { logger.warn("skipping, no node available"); } finally { client.close(); - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java index 7a2c3fc..f121209 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,24 +35,27 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_prune1", settings); - client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); - client.newIndex("test_prune2", settings); - client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); - client.newIndex("test_prune3", settings); - client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); - client.newIndex("test_prune4", settings); - client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); + bulkClient.newIndex("test_prune1", settings); + adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + bulkClient.newIndex("test_prune2", settings); + adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + bulkClient.newIndex("test_prune3", settings); + adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + bulkClient.newIndex("test_prune4", settings); + adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test_prune", "test_prune4", 2, 2, true); + adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); @@ -62,7 +63,7 @@ class IndexPruneTest { List list = new ArrayList<>(); for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(new String[] { index }); + indicesExistsRequest.indices(index); IndicesExistsResponse indicesExistsResponse = client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); list.add(indicesExistsResponse.isExists()); @@ -72,14 +73,13 @@ class IndexPruneTest { assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java index 87d422c..0fdec74 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java @@ -3,16 +3,16 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -34,52 +34,55 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) + .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1234", settings); + client.newIndex("test_shift", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", helper.randomString(1), false, + client.index("test_shift", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test_shift", "test1234", Arrays.asList("a", "b", "c")); + client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - Map aliases = client.getAliases("test1234"); + Map aliases = client.getAliases("test_shift"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("test_shift")); + assertTrue(aliases.containsKey("test")); - String resolved = client.resolveAlias("test_shift"); + String resolved = client.resolveAlias("test"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); - assertTrue(aliases.containsKey("test_shift")); + assertTrue(aliases.containsKey("test")); - client.newIndex("test5678", settings); + client.newIndex("test_shift2", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", helper.randomString(1), false, + client.index("test_shift2", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - indexShiftResult = client.shiftIndex("test_shift", "test5678", Arrays.asList("d", "e", "f"), - (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - index, alias).filter(QueryBuilders.termQuery("my_key", alias))) + indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"), + (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); assertTrue(indexShiftResult.getNewAliases().contains("d")); assertTrue(indexShiftResult.getNewAliases().contains("e")); @@ -88,7 +91,7 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - aliases = client.getAliases("test5678"); + aliases = client.getAliases("test_shift2"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); @@ -96,7 +99,7 @@ class IndexShiftTest { assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - resolved = client.resolveAlias("test_shift"); + resolved = client.resolveAlias("test"); aliases = client.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); @@ -104,15 +107,13 @@ class IndexShiftTest { assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java index c0bab0d..76593c6 100644 --- a/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java +++ b/elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.node.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.api.IndexDefinition; -import org.xbib.elx.node.ExtendedNodeClient; -import org.xbib.elx.node.ExtendedNodeClientProvider; +import org.xbib.elx.node.NodeAdminClient; +import org.xbib.elx.node.NodeAdminClientProvider; +import org.xbib.elx.node.NodeBulkClient; +import org.xbib.elx.node.NodeBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -29,41 +30,56 @@ class SmokeTest { @Test void smokeTest() throws Exception { - final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1")) - .provider(ExtendedNodeClientProvider.class) + final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1")) + .setAdminClientProvider(NodeAdminClientProvider.class) .build(); + final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1")) + .setBulkClientProvider(NodeBulkClientProvider.class) + .build(); + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); try { + assertEquals(helper.getClusterName(), client.getClusterName()); client.newIndex("test_smoke"); client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - assertEquals(helper.getClusterName(), client.getClusterName()); client.checkMapping("test_smoke"); client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); + client.delete("test_smoke", "1"); client.flush(); + client.waitForResponses(30, TimeUnit.SECONDS); client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); + client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); client.delete("test_smoke", "1"); + client.flush(); client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke_2", Settings.settingsBuilder() + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); client.newIndex(indexDefinition); client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.flush(); + client.waitForResponses(30, TimeUnit.SECONDS); client.updateReplicaLevel(indexDefinition, 2); int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); - assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount()); + //assertEquals(0, client.getBulkMetric().getFailed().getCount()); + //assertEquals(6, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + // close admin after bulk + adminClient.deleteIndex(indexDefinition); + adminClient.close(); } } } diff --git a/elx-node/src/test/resources/log4j2.xml b/elx-node/src/test/resources/log4j2-test.xml similarity index 100% rename from elx-node/src/test/resources/log4j2.xml rename to elx-node/src/test/resources/log4j2-test.xml diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java deleted file mode 100644 index c50bbd4..0000000 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.xbib.elx.transport; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; -import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; -import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; -import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.InetSocketTransportAddress; -import org.jboss.netty.channel.DefaultChannelFuture; -import org.xbib.elx.common.AbstractExtendedClient; -import org.xbib.elx.common.util.NetworkUtils; - -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Transport client with additional methods using the BulkProcessor. - */ -public class ExtendedTransportClient extends AbstractExtendedClient { - - private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName()); - - @Override - protected ElasticsearchClient createClient(Settings settings) { - if (settings != null) { - String systemIdentifier = System.getProperty("os.name") - + " " + System.getProperty("java.vm.name") - + " " + System.getProperty("java.vm.vendor") - + " " + System.getProperty("java.vm.version") - + " Elasticsearch " + Version.CURRENT.toString(); - Settings effectiveSettings = Settings.builder() - // for thread pool size - .put("processors", - settings.getAsInt("processors", Runtime.getRuntime().availableProcessors())) - .put("client.transport.sniff", false) // do not sniff - .put("client.transport.nodes_sampler_interval", "1m") // do not ping - .put("client.transport.ping_timeout", "1m") // wait for unresponsive nodes a very long time before disconnect - .put("client.transport.ignore_cluster_name", true) // connect to any cluster - // custom settings may override defaults - .put(settings) - .build(); - logger.info("creating transport client on {} with custom settings {} and effective settings {}", - systemIdentifier, settings.getAsMap(), effectiveSettings.getAsMap()); - // we need to disable dead lock check because we may have mixed node/transport clients - DefaultChannelFuture.setUseDeadLockChecker(false); - return TransportClient.builder().settings(effectiveSettings).build(); - } - return null; - } - - @Override - protected void closeClient() { - if (getClient() != null) { - TransportClient client = (TransportClient) getClient(); - client.close(); - client.threadPool().shutdown(); - } - } - - @Override - public ExtendedTransportClient init(Settings settings) throws IOException { - super.init(settings); - // additional auto-connect - try { - Collection addrs = findAddresses(settings); - if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { - throw new NoNodeAvailableException("no cluster nodes available, check settings " - + settings.toString()); - } - } catch (IOException e) { - logger.error(e.getMessage(), e); - } - return this; - } - - private Collection findAddresses(Settings settings) throws IOException { - final int defaultPort = settings.getAsInt("port", 9300); - Collection addresses = new ArrayList<>(); - for (String hostname : settings.getAsArray("host")) { - String[] splitHost = hostname.split(":", 2); - if (splitHost.length == 2) { - try { - String host = splitHost[0]; - InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); - int port = Integer.parseInt(splitHost[1]); - InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, port); - addresses.add(address); - } catch (NumberFormatException e) { - logger.warn(e.getMessage(), e); - } - } - if (splitHost.length == 1) { - String host = splitHost[0]; - InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); - InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, defaultPort); - addresses.add(address); - } - } - return addresses; - } - - private boolean connect(Collection addresses, boolean autodiscover) { - if (getClient() == null) { - throw new IllegalStateException("no client present"); - } - logger.debug("trying to connect to {}", addresses); - TransportClient transportClient = (TransportClient) getClient(); - transportClient.addTransportAddresses(addresses); - List nodes = transportClient.connectedNodes(); - logger.info("connected to nodes = {}", nodes); - if (nodes != null && !nodes.isEmpty()) { - if (autodiscover) { - logger.debug("trying to auto-discover all nodes..."); - ClusterStateRequestBuilder clusterStateRequestBuilder = - new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE); - ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); - DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); - transportClient.addDiscoveryNodes(discoveryNodes); - logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes()); - } - return true; - } - return false; - } -} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java deleted file mode 100644 index 669d21a..0000000 --- a/elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.xbib.elx.transport; - -import org.xbib.elx.api.ExtendedClientProvider; - -public class ExtendedTransportClientProvider implements ExtendedClientProvider { - - @Override - public ExtendedTransportClient getExtendedClient() { - return new ExtendedTransportClient(); - } -} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java new file mode 100644 index 0000000..3987f63 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java @@ -0,0 +1,40 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractAdminClient; + +import java.io.IOException; + +/** + * Transport admin client. + */ +public class TransportAdminClient extends AbstractAdminClient { + + private final TransportClientHelper helper; + + public TransportAdminClient() { + this.helper = new TransportClientHelper(); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java new file mode 100644 index 0000000..18d2368 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.AdminClientProvider; + +public class TransportAdminClientProvider implements AdminClientProvider { + + @Override + public TransportAdminClient getClient() { + return new TransportAdminClient(); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java new file mode 100644 index 0000000..ebc8621 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java @@ -0,0 +1,39 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractBulkClient; +import java.io.IOException; + +/** + * Transport search client with additional methods. + */ +public class TransportBulkClient extends AbstractBulkClient { + + private final TransportClientHelper helper; + + public TransportBulkClient() { + this.helper = new TransportClientHelper(); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java new file mode 100644 index 0000000..399d73d --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.BulkClientProvider; + +public class TransportBulkClientProvider implements BulkClientProvider { + + @Override + public TransportBulkClient getClient() { + return new TransportBulkClient(); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java new file mode 100644 index 0000000..4ac9ff2 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java @@ -0,0 +1,161 @@ +package org.xbib.elx.transport; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateAction; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.network.NetworkModule; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.netty4.Netty4Plugin; +import org.xbib.elx.common.util.NetworkUtils; +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class TransportClientHelper { + + + private static final Logger logger = LogManager.getLogger(TransportAdminClient.class.getName()); + + protected ElasticsearchClient createClient(Settings settings) throws IOException { + if (settings != null) { + String systemIdentifier = System.getProperty("os.name") + + " " + System.getProperty("java.vm.name") + + " " + System.getProperty("java.vm.vendor") + + " " + System.getProperty("java.vm.version") + + " Elasticsearch " + Version.CURRENT.toString(); + Settings transportClientSettings = getTransportClientSettings(settings); + //XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject(); + XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject(); + logger.log(Level.INFO, "creating transport client on {} with settings {}", + systemIdentifier, + //Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()), + Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder, + ToXContent.EMPTY_PARAMS).endObject())); + return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class)); + } + return null; + } + + public void init(TransportClient transportClient, Settings settings) throws IOException { + Collection addrs = findAddresses(settings); + if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.toString()); + } + } + + @Override + public ExtendedTransportClient init(Settings settings) throws IOException { + super.init(settings); + // additional auto-connect + try { + Collection addrs = findAddresses(settings); + if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) { + throw new NoNodeAvailableException("no cluster nodes available, check settings " + + settings.toString()); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + return this; + } + + private Collection findAddresses(Settings settings) throws IOException { + final int defaultPort = settings.getAsInt("port", 9300); + Collection addresses = new ArrayList<>(); + for (String hostname : settings.getAsList("host")) { + String[] splitHost = hostname.split(":", 2); + if (splitHost.length == 2) { + try { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + int port = Integer.parseInt(splitHost[1]); + TransportAddress address = new TransportAddress(inetAddress, port); + addresses.add(address); + } catch (NumberFormatException e) { + logger.warn(e.getMessage(), e); + } + } + if (splitHost.length == 1) { + String host = splitHost[0]; + InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null); + TransportAddress address = new TransportAddress(inetAddress, defaultPort); + addresses.add(address); + } + } + return addresses; + } + + private boolean connect(Collection addresses, boolean autodiscover) { + if (getClient() == null) { + throw new IllegalStateException("no client present"); + } + TransportClient transportClient = (TransportClient) getClient(); + for (TransportAddress address : addresses) { + transportClient.addTransportAddresses(address); + } + List nodes = transportClient.connectedNodes(); + logger.info("connected to nodes = {}", nodes); + if (nodes != null && !nodes.isEmpty()) { + if (autodiscover) { + logger.debug("trying to auto-discover all nodes..."); + ClusterStateRequestBuilder clusterStateRequestBuilder = + new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE); + ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet(); + DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes(); + addDiscoveryNodes(transportClient, discoveryNodes); + logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes()); + } + return true; + } + return false; + } + + private Settings getTransportClientSettings(Settings settings) { + return Settings.builder() + // "cluster.name" + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), + settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey())) + // "processors" + .put(EsExecutors.PROCESSORS_SETTING.getKey(), + settings.get(EsExecutors.PROCESSORS_SETTING.getKey(), + String.valueOf(Runtime.getRuntime().availableProcessors()))) + // "transport.type" + .put(NetworkModule.TRANSPORT_TYPE_KEY, + Netty4Plugin.NETTY_TRANSPORT_NAME) + .build(); + } + + private void addDiscoveryNodes(TransportClient transportClient, DiscoveryNodes discoveryNodes) { + for (DiscoveryNode discoveryNode : discoveryNodes) { + transportClient.addTransportAddress(discoveryNode.getAddress()); + } + } + + static class MyTransportClient extends TransportClient { + + MyTransportClient(Settings settings, Collection> plugins) { + super(settings, plugins); + } + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java new file mode 100644 index 0000000..750914c --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java @@ -0,0 +1,39 @@ +package org.xbib.elx.transport; + +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; +import org.xbib.elx.common.AbstractSearchClient; +import java.io.IOException; + +/** + * Transport search client with additional methods. + */ +public class TransportSearchClient extends AbstractSearchClient { + + private final TransportClientHelper helper; + + public TransportSearchClient() { + this.helper = new TransportClientHelper(); + } + + @Override + protected ElasticsearchClient createClient(Settings settings) throws IOException { + return helper.createClient(settings); + } + + @Override + protected void closeClient() { + if (getClient() != null) { + TransportClient client = (TransportClient) getClient(); + client.close(); + client.threadPool().shutdown(); + } + } + + @Override + public void init(Settings settings) throws IOException { + super.init(settings); + helper.init((TransportClient) getClient(), settings); + } +} diff --git a/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java new file mode 100644 index 0000000..c77e320 --- /dev/null +++ b/elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java @@ -0,0 +1,11 @@ +package org.xbib.elx.transport; + +import org.xbib.elx.api.SearchClientProvider; + +public class TransportSearchClientProvider implements SearchClientProvider { + + @Override + public TransportSearchClient getClient() { + return new TransportSearchClient(); + } +} diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider new file mode 100644 index 0000000..37ce7ac --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportAdminClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider new file mode 100644 index 0000000..e1c69bd --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportBulkClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider deleted file mode 100644 index 640e2f9..0000000 --- a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider +++ /dev/null @@ -1 +0,0 @@ -org.xbib.elx.transport.ExtendedTransportClientProvider \ No newline at end of file diff --git a/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider new file mode 100644 index 0000000..ae6a33b --- /dev/null +++ b/elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider @@ -0,0 +1 @@ +org.xbib.elx.transport.TransportSearchClientProvider \ No newline at end of file diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java new file mode 100644 index 0000000..388f6f3 --- /dev/null +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java @@ -0,0 +1,188 @@ +package org.xbib.elx.transport.test; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.transport.NoNodeAvailableException; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.xbib.elx.common.ClientBuilder; +import org.xbib.elx.common.Parameters; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(TestExtension.class) +class BulkClientTest { + + private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName()); + + private static final Long ACTIONS = 10000L; + + private static final Long MAX_ACTIONS_PER_REQUEST = 10000L; + + private final TestExtension.Helper helper; + + BulkClientTest(TestExtension.Helper helper) { + this.helper = helper; + } + + @Test + void testSingleDoc() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30)) + .build(); + try { + bulkClient.newIndex("test"); + bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } finally { + assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.close(); + } + } + + @Test + void testNewIndex() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) + .build(); + bulkClient.newIndex("test"); + bulkClient.close(); + } + + @Test + void testMapping() throws Exception { + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("doc") + .startObject("properties") + .startObject("location") + .field("type", "geo_point") + .endObject() + .endObject() + .endObject() + .endObject(); + bulkClient.newIndex("test", Settings.EMPTY, builder); + assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties")); + bulkClient.close(); + } + + @Test + void testRandomDocs() throws Exception { + long numactions = ACTIONS; + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .build(); + try { + bulkClient.newIndex("test"); + for (int i = 0; i < ACTIONS; i++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(numactions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } + + @Test + void testThreadedRandomDocs() throws Exception { + int maxthreads = Runtime.getRuntime().availableProcessors(); + Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST; + final long actions = ACTIONS; + logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2) + .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest) + .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) + .put(Parameters.ENABLE_BULK_LOGGING.name(), "true") + .build(); + try { + Settings settings = Settings.builder() + .put("index.number_of_shards", 1) + .put("index.number_of_replicas", 0) + .build(); + bulkClient.newIndex("test", settings); + bulkClient.startBulk("test", 0, 1000); + logger.info("index created"); + ExecutorService executorService = Executors.newFixedThreadPool(maxthreads); + final CountDownLatch latch = new CountDownLatch(maxthreads); + for (int i = 0; i < maxthreads; i++) { + executorService.execute(() -> { + for (int i1 = 0; i1 < actions; i1++) { + bulkClient.index("test", null, false, + "{ \"name\" : \"" + helper.randomString(32) + "\"}"); + } + latch.countDown(); + }); + } + logger.info("waiting for latch..."); + if (latch.await(30L, TimeUnit.SECONDS)) { + logger.info("flush..."); + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); + logger.info("got all responses, executor service shutdown..."); + executorService.shutdown(); + executorService.awaitTermination(30L, TimeUnit.SECONDS); + logger.info("pool is shut down"); + } else { + logger.warn("latch timeout"); + } + bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS); + assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount()); + } catch (NoNodeAvailableException e) { + logger.warn("skipping, no node available"); + } catch (Exception e) { + logger.error(e.getMessage(), e); + } finally { + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); + } + assertNull(bulkClient.getBulkController().getLastBulkError()); + bulkClient.refreshIndex("test"); + assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test")); + bulkClient.close(); + } + } +} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java deleted file mode 100644 index 409f38f..0000000 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.xbib.elx.transport.test; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.action.search.SearchAction; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.common.Parameters; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@ExtendWith(TestExtension.class) -class ClientTest { - - private static final Logger logger = LogManager.getLogger(ClientTest.class.getName()); - - private static final Long ACTIONS = 100L; - - private static final Long MAX_ACTIONS_PER_REQUEST = 1000L; - - private final TestExtension.Helper helper; - - ClientTest(TestExtension.Helper helper) { - this.helper = helper; - } - - @Test - void testClientIndexOp() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - client.newIndex("test1"); - client.close(); - } - - @Test - void testMapping() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5)) - .build(); - XContentBuilder builder = jsonBuilder() - .startObject() - .startObject("doc") - .startObject("properties") - .startObject("location") - .field("type", "geo_point") - .endObject() - .endObject() - .endObject() - .endObject(); - client.newIndex("test2", Settings.EMPTY, builder.string()); - GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2"); - GetMappingsResponse getMappingsResponse = - client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet(); - logger.info("mappings={}", getMappingsResponse.getMappings()); - assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc")); - client.close(); - } - - @Test - void testSingleDoc() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test3"); - client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}"); - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.close(); - } - } - - @Test - void testRandomDocs() throws Exception { - long numactions = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test4"); - for (int i = 0; i < ACTIONS; i++) { - client.index("test4", null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - client.flush(); - client.waitForResponses(60L, TimeUnit.SECONDS); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test4"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test4") - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0); - assertEquals(numactions, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.close(); - } - } - - @Test - void testThreadedRandomDocs() throws Exception { - int maxthreads = Runtime.getRuntime().availableProcessors(); - long maxactions = MAX_ACTIONS_PER_REQUEST; - final long maxloop = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) - .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions) - .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60)) - .build(); - try { - client.newIndex("test5") - .startBulk("test5", -1, 1000); - ThreadPoolExecutor pool = EsExecutors.newFixed("transportclient-test", maxthreads, 30, - EsExecutors.daemonThreadFactory("transportclient-test")); - final CountDownLatch latch = new CountDownLatch(maxthreads); - for (int i = 0; i < maxthreads; i++) { - pool.execute(() -> { - for (int i1 = 0; i1 < maxloop; i1++) { - client.index("test5",null, false, - "{ \"name\" : \"" + helper.randomString(32) + "\"}"); - } - latch.countDown(); - }); - } - logger.info("waiting for latch..."); - if (latch.await(60, TimeUnit.SECONDS)) { - logger.info("flush ..."); - client.flush(); - client.waitForResponses(60L, TimeUnit.SECONDS); - logger.info("pool shutdown ..."); - pool.shutdown(); - logger.info("poot is shut down"); - } else { - logger.warn("latch timeout"); - } - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); - } finally { - client.stopBulk("test5", 60L, TimeUnit.SECONDS); - assertEquals(maxthreads * maxloop, client.getBulkController().getBulkMetric().getSucceeded().getCount()); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); - } - assertNull(client.getBulkController().getLastBulkError()); - client.refreshIndex("test5"); - SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE) - .setIndices("test5") - .setQuery(QueryBuilders.matchAllQuery()) - .setSize(0); - assertEquals(maxthreads * maxloop, - searchRequestBuilder.execute().actionGet().getHits().getTotalHits()); - client.close(); - } - } -} diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java index 37149f6..c023ef5 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -11,8 +12,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.common.ClientBuilder; import org.xbib.elx.common.Parameters; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -23,11 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; @ExtendWith(TestExtension.class) class DuplicateIDTest { - private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); + private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName()); - private final static Long MAX_ACTIONS_PER_REQUEST = 100L; + private static final Long MAX_ACTIONS_PER_REQUEST = 10L; - private final static Long ACTIONS = 50L; + private static final Long ACTIONS = 5L; private final TestExtension.Helper helper; @@ -38,15 +39,15 @@ class DuplicateIDTest { @Test void testDuplicateDocIDs() throws Exception { long numactions = ACTIONS; - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()) + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST) + .put(helper.getTransportSettings()) .build(); try { - client.newIndex("test_dup"); + bulkClient.newIndex("test_dup"); for (int i = 0; i < ACTIONS; i++) { - client.index("test_dup", helper.randomString(1), false, + bulkClient.index("test_dup", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); @@ -54,21 +55,25 @@ class DuplicateIDTest { client.refreshIndex("test_dup"); SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); + builder.size(0); SearchRequest searchRequest = new SearchRequest(); searchRequest.indices("test_dup"); + searchRequest.types("test_dup"); searchRequest.source(builder); - long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits(); + SearchResponse searchResponse = + helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet(); + long hits = searchResponse.getHits().getTotalHits(); logger.info("hits = {}", hits); assertTrue(hits < ACTIONS); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { client.close(); - assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount()); + assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount()); if (client.getBulkController().getLastBulkError() != null) { logger.error("error", client.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java index de886d1..3304e4e 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java @@ -2,17 +2,15 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; -import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexPruneResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.io.IOException; import java.util.ArrayList; @@ -37,8 +35,12 @@ class IndexPruneTest { @Test void testPrune() throws IOException { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) .put(helper.getTransportSettings()) .build(); try { @@ -46,27 +48,24 @@ class IndexPruneTest { .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test_prune1", settings); - client.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); - client.newIndex("test_prune2", settings); - client.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); - client.newIndex("test_prune3", settings); - client.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); - client.newIndex("test_prune4", settings); - client.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); - + bulkClient.newIndex("test_prune1", settings); + adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList()); + bulkClient.newIndex("test_prune2", settings); + adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList()); + bulkClient.newIndex("test_prune3", settings); + adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList()); + bulkClient.newIndex("test_prune4", settings); + adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList()); IndexPruneResult indexPruneResult = - client.pruneIndex("test_prune", "test_prune4", 2, 2, true); - + adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1")); assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3")); assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4")); - List list = new ArrayList<>(); for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) { IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest(); - indicesExistsRequest.indices(new String[] { index }); + indicesExistsRequest.indices(index); IndicesExistsResponse indicesExistsResponse = client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet(); list.add(indicesExistsResponse.isExists()); @@ -76,14 +75,13 @@ class IndexPruneTest { assertFalse(list.get(1)); assertTrue(list.get(2)); assertTrue(list.get(3)); - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java index 1f65064..0b7e126 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java @@ -3,16 +3,16 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; -import org.elasticsearch.client.transport.NoNodeAvailableException; -import org.elasticsearch.cluster.metadata.AliasAction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryBuilders; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexShiftResult; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.Arrays; import java.util.Map; @@ -34,54 +34,53 @@ class IndexShiftTest { @Test void testIndexShift() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) - .put(helper.getTransportSettings()).build(); + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build(); try { Settings settings = Settings.builder() .put("index.number_of_shards", 1) .put("index.number_of_replicas", 0) .build(); - client.newIndex("test1234", settings); + client.newIndex("test_shift1234", settings); for (int i = 0; i < 1; i++) { - client.index("test1234", helper.randomString(1), false, + client.index("test_shift1234", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } - client.flush(); - client.waitForResponses(30L, TimeUnit.SECONDS); - + bulkClient.flush(); + bulkClient.waitForResponses(30L, TimeUnit.SECONDS); IndexShiftResult indexShiftResult = - client.shiftIndex("test_shift", "test1234", Arrays.asList("a", "b", "c")); - + client.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c")); assertTrue(indexShiftResult.getNewAliases().contains("a")); assertTrue(indexShiftResult.getNewAliases().contains("b")); assertTrue(indexShiftResult.getNewAliases().contains("c")); assertTrue(indexShiftResult.getMovedAliases().isEmpty()); - - Map aliases = client.getAliases("test1234"); + Map aliases = client.getAliases("test_shift1234"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test_shift")); - - String resolved = client.resolveAlias("test_shift"); - aliases = client.getAliases(resolved); + String resolved = adminClient.resolveAlias("test_shift"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("test_shift")); - - client.newIndex("test5678", settings); + client.newIndex("test_shift5678", settings); for (int i = 0; i < 1; i++) { - client.index("test5678", helper.randomString(1), false, + client.index("test_shift5678", helper.randomString(1), false, "{ \"name\" : \"" + helper.randomString(32) + "\"}"); } client.flush(); client.waitForResponses(30L, TimeUnit.SECONDS); - - indexShiftResult = client.shiftIndex("test_shift", "test5678", Arrays.asList("d", "e", "f"), - (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD, - index, alias).filter(QueryBuilders.termQuery("my_key", alias))) + indexShiftResult = client.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"), + (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add() + .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias))) ); assertTrue(indexShiftResult.getNewAliases().contains("d")); assertTrue(indexShiftResult.getNewAliases().contains("e")); @@ -89,32 +88,28 @@ class IndexShiftTest { assertTrue(indexShiftResult.getMovedAliases().contains("a")); assertTrue(indexShiftResult.getMovedAliases().contains("b")); assertTrue(indexShiftResult.getMovedAliases().contains("c")); - - aliases = client.getAliases("test5678"); + aliases = client.getAliases("test_shift5678"); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - resolved = client.resolveAlias("test_shift"); - aliases = client.getAliases(resolved); + resolved = adminClient.resolveAlias("test_shift"); + aliases = adminClient.getAliases(resolved); assertTrue(aliases.containsKey("a")); assertTrue(aliases.containsKey("b")); assertTrue(aliases.containsKey("c")); assertTrue(aliases.containsKey("d")); assertTrue(aliases.containsKey("e")); assertTrue(aliases.containsKey("f")); - - } catch (NoNodeAvailableException e) { - logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + adminClient.close(); + bulkClient.close(); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); } } } diff --git a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java index 74d8dee..b0fb265 100644 --- a/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java +++ b/elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java @@ -2,14 +2,15 @@ package org.xbib.elx.transport.test; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.transport.NoNodeAvailableException; import org.elasticsearch.common.settings.Settings; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.xbib.elx.api.IndexDefinition; import org.xbib.elx.common.ClientBuilder; -import org.xbib.elx.transport.ExtendedTransportClient; -import org.xbib.elx.transport.ExtendedTransportClientProvider; +import org.xbib.elx.transport.TransportAdminClient; +import org.xbib.elx.transport.TransportAdminClientProvider; +import org.xbib.elx.transport.TransportBulkClient; +import org.xbib.elx.transport.TransportBulkClientProvider; import java.util.concurrent.TimeUnit; @@ -17,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @ExtendWith(TestExtension.class) -class SmokeTest extends TestExtension { +class SmokeTest { private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName()); @@ -28,44 +29,56 @@ class SmokeTest extends TestExtension { } @Test - void testSingleDocNodeClient() throws Exception { - final ExtendedTransportClient client = ClientBuilder.builder() - .provider(ExtendedTransportClientProvider.class) + void smokeTest() throws Exception { + final TransportAdminClient adminClient = ClientBuilder.builder() + .setAdminClientProvider(TransportAdminClientProvider.class) .put(helper.getTransportSettings()) .build(); + final TransportBulkClient bulkClient = ClientBuilder.builder() + .setBulkClientProvider(TransportBulkClientProvider.class) + .put(helper.getTransportSettings()) + .build(); + IndexDefinition indexDefinition = + adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY); try { + assertEquals(helper.getClusterName(), client.getClusterName()); client.newIndex("test_smoke"); client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest client.flush(); client.waitForResponses(30, TimeUnit.SECONDS); - assertEquals(helper.getClusterName(), client.getClusterName()); client.checkMapping("test_smoke"); client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}"); - client.flush(); - client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS); client.delete("test_smoke", "1"); + client.flush(); + client.waitForResponses(30, TimeUnit.SECONDS); + client.checkMapping("test_smoke"); client.deleteIndex("test_smoke"); - IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke_2", Settings.settingsBuilder() + IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder() .build()); assertEquals(0, indexDefinition.getReplicaLevel()); client.newIndex(indexDefinition); client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}"); client.flush(); + client.waitForResponses(30, TimeUnit.SECONDS); client.updateReplicaLevel(indexDefinition, 2); int replica = client.getReplicaLevel(indexDefinition); assertEquals(2, replica); - client.deleteIndex(indexDefinition); - assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount()); - assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount()); + assertEquals(0, client.getBulkMetric().getFailed().getCount()); + assertEquals(4, client.getBulkMetric().getSucceeded().getCount()); } catch (NoNodeAvailableException e) { logger.warn("skipping, no node available"); } finally { - client.close(); - if (client.getBulkController().getLastBulkError() != null) { - logger.error("error", client.getBulkController().getLastBulkError()); + bulkClient.close(); + assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount()); + assertEquals(4, bulkClient.getBulkMetric().getSucceeded().getCount()); + if (bulkClient.getBulkController().getLastBulkError() != null) { + logger.error("error", bulkClient.getBulkController().getLastBulkError()); } - assertNull(client.getBulkController().getLastBulkError()); + assertNull(bulkClient.getBulkController().getLastBulkError()); + // close admin after bulk + adminClient.deleteIndex(indexDefinition); + adminClient.close(); } } }