split into admin, bulk, search client, fix HTTP client

build.gradle

@@ -11,7 +11,7 @@ wrapper {
 ext {
     user = 'jprante'
     name = 'elx'
-    description = 'Extensions for Elasticsearch clients (node and transport)'
+    description = 'Admin/Bulk/Search API extensions for Elasticsearch clients (node, transport, http)'
     inceptionYear = '2019'
     url = 'https://github.com/' + user + '/' + name
     scmUrl = 'https://github.com/' + user + '/' + name

elx-api/src/main/java/org/xbib/elx/api/AdminClient.java

@@ -0,0 +1,194 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Interface for extended managing and indexing methods of an Elasticsearch client.
+ */
+public interface AdminClient extends NativeClient {
+
+    /**
+     * Build index definition from settings.
+     *
+     * @param index the index name
+     * @param settings the settings for the index
+     * @return index definition
+     * @throws IOException if settings/mapping URL is invalid/malformed
+     */
+    IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
+
+    /**
+     * Delete an index.
+     * @param indexDefinition the index definition
+     * @return this
+     */
+    AdminClient deleteIndex(IndexDefinition indexDefinition);
+
+    /**
+     * Delete an index.
+     *
+     * @param index index
+     * @return this
+     */
+    AdminClient deleteIndex(String index);
+
+
+    /**
+     * Update replica level.
+     * @param indexDefinition the index definition
+     * @param level the replica level
+     * @return this
+     * @throws IOException if replica setting could not be updated
+     */
+    AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
+
+    /**
+     * Update replica level.
+     *
+     * @param index index
+     * @param level the replica level
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return this
+     * @throws IOException if replica setting could not be updated
+     */
+    AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
+
+    /**
+     * Get replica level.
+     * @param indexDefinition the index name
+     * @return the replica level of the index
+     */
+    int getReplicaLevel(IndexDefinition indexDefinition);
+
+    /**
+     * Get replica level.
+     * @param index the index name
+     * @return the replica level of the index
+     */
+    int getReplicaLevel(String index);
+
+    /**
+     * Force segment merge of an index.
+     * @param indexDefinition the index definition
+     * @return this
+     */
+    boolean forceMerge(IndexDefinition indexDefinition);
+
+    /**
+     * Force segment merge of an index.
+     * @param index the index
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return this
+     */
+    boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
+
+
+    /**
+     * Wait for index recovery (after replica change).
+     *
+     * @param index index
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return true if wait succeeded, false if wait timed out
+     */
+    boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
+
+    /**
+     * Resolve alias.
+     *
+     * @param alias the alias
+     * @return this index name behind the alias or the alias if there is no index
+     */
+    String resolveAlias(String alias);
+
+    /**
+     * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
+     * name.
+     *
+     * @param alias the alias
+     * @return the most recent index name pointing to the alias
+     */
+    String resolveMostRecentIndex(String alias);
+
+    /**
+     * Get all index aliases.
+     * @param index the index
+     * @return map of index aliases
+     */
+    Map<String, String> getAliases(String index);
+
+    /**
+     *  Shift from one index to another.
+     * @param indexDefinition the index definition
+     * @param additionalAliases new aliases
+     * @return this
+     */
+    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
+
+    /**
+     * Shift from one index to another.
+     * @param indexDefinition the index definition
+     * @param additionalAliases new aliases
+     * @param indexAliasAdder method to add aliases
+     * @return this
+     */
+    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
+                                IndexAliasAdder indexAliasAdder);
+
+    /**
+     * Shift from one index to another.
+     * @param index         the index name
+     * @param fullIndexName the index name with timestamp
+     * @param additionalAliases  a list of names that should be set as index aliases
+     * @return this
+     */
+    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
+
+    /**
+     * Shift from one index to another.
+     * @param index         the index name
+     * @param fullIndexName the index name with timestamp
+     * @param additionalAliases  a list of names that should be set as index aliases
+     * @param adder         an adder method to create alias term queries
+     * @return this
+     */
+    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
+                                IndexAliasAdder adder);
+
+    /**
+     * Prune index.
+     * @param indexDefinition the index definition
+     * @return the index prune result
+     */
+    IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
+
+    /**
+     * Apply retention policy to prune indices. All indices before delta should be deleted,
+     * but the number of mintokeep indices must be kept.
+     *
+     * @param index         index name
+     * @param fullIndexName index name with timestamp
+     * @param delta timestamp delta (for index timestamps)
+     * @param mintokeep     minimum number of indices to keep
+     * @param perform true if pruning should be executed, false if not
+     * @return the index prune result
+     */
+    IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
+
+    /**
+     * Find the timestamp of the most recently indexed document in the index.
+     *
+     * @param index the index name
+     * @param timestampfieldname the timestamp field name
+     * @return millis UTC millis of the most recent document
+     * @throws IOException if most rcent document can not be found
+     */
+    Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
+}

elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java

@@ -0,0 +1,7 @@
+package org.xbib.elx.api;
+
+@FunctionalInterface
+public interface AdminClientProvider<C extends AdminClient> {
+
+    C getClient();
+}

elx-api/src/main/java/org/xbib/elx/api/BulkClient.java

@@ -0,0 +1,227 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import java.io.Flushable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public interface BulkClient extends NativeClient, Flushable {
+
+    /**
+     * Get bulk metric.
+     * @return the bulk metric
+     */
+    BulkMetric getBulkMetric();
+
+    /**
+     * Get buulk control.
+     * @return the bulk control
+     */
+    BulkController getBulkController();
+
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @throws IOException if new index creation fails
+     */
+    void newIndex(String index) throws IOException;
+
+    /**
+     * Create a new index.
+     * @param indexDefinition the index definition
+     * @throws IOException if settings/mapping is invalid or index creation fails
+     */
+    void newIndex(IndexDefinition indexDefinition) throws IOException;
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @param settings settings
+     * @throws IOException if settings is invalid or index creation fails
+     */
+    void newIndex(String index, Settings settings) throws IOException;
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @param settings settings
+     * @param mapping mapping
+     * @throws IOException if settings/mapping is invalid or index creation fails
+     */
+    void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @param settings settings
+     * @param mapping mapping
+     * @throws IOException if settings/mapping is invalid or index creation fails
+     */
+    void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
+
+    /**
+     * Add index request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when limits are exceeded.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param create true if document must be created
+     * @param source the source
+     * @return this
+     */
+    BulkClient index(String index, String id, boolean create, BytesReference source);
+
+    /**
+     * Index request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when limits are exceeded.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param create true if document is to be created, false otherwise
+     * @param source the source
+     * @return this client methods
+     */
+    BulkClient index(String index, String id, boolean create, String source);
+
+    /**
+     * Index request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     *
+     * @param indexRequest the index request to add
+     * @return this
+     */
+    BulkClient index(IndexRequest indexRequest);
+
+    /**
+     * Delete request.
+     *
+     * @param index the index
+     * @param id    the id
+     * @return this
+     */
+    BulkClient delete(String index, String id);
+
+    /**
+     * Delete request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     *
+     * @param deleteRequest the delete request to add
+     * @return this
+     */
+    BulkClient delete(DeleteRequest deleteRequest);
+
+    /**
+     * Bulked update request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     * Note that updates only work correctly when all operations between nodes are synchronized.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param source the source
+     * @return this
+     */
+    BulkClient update(String index, String id, BytesReference source);
+
+    /**
+     * Update document. Use with precaution! Does not work in all cases.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param source the source
+     * @return this
+     */
+    BulkClient update(String index, String id, String source);
+
+    /**
+     * Bulked update request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     * Note that updates only work correctly when all operations between nodes are synchronized.
+     *
+     * @param updateRequest the update request to add
+     * @return this
+     */
+    BulkClient update(UpdateRequest updateRequest);
+
+    /**
+     * Start bulk mode for indexes.
+     * @param indexDefinition index definition
+     * @throws IOException if bulk could not be started
+     */
+    void startBulk(IndexDefinition indexDefinition) throws IOException;
+
+    /**
+     * Start bulk mode.
+     *
+     * @param index index
+     * @param startRefreshIntervalSeconds refresh interval before bulk
+     * @param stopRefreshIntervalSeconds  refresh interval after bulk
+     * @throws IOException if bulk could not be started
+     */
+    void startBulk(String index, long startRefreshIntervalSeconds,
+                          long stopRefreshIntervalSeconds) throws IOException;
+
+
+    /**
+     * Stop bulk mode.
+     *
+     * @param indexDefinition index definition
+     * @throws IOException if bulk could not be startet
+     */
+    void stopBulk(IndexDefinition indexDefinition) throws IOException;
+
+    /**
+     * Stops bulk mode.
+     *
+     * @param index index
+     * @param timeout maximum wait time
+     * @param timeUnit time unit for timeout
+     * @throws IOException if bulk could not be stopped
+     */
+    void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
+
+    /**
+     * Wait for all outstanding bulk responses.
+     *
+     * @param timeout maximum wait time
+     * @param timeUnit unit of timeout value
+     * @return true if wait succeeded, false if wait timed out
+     */
+    boolean waitForResponses(long timeout, TimeUnit timeUnit);
+
+    /**
+     * Update index setting.
+     * @param index the index
+     * @param key the key of the value to be updated
+     * @param value the new value
+     * @param timeout timeout
+     * @param timeUnit time unit
+     * @throws IOException if update index setting failed
+     */
+    void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
+
+    /**
+     * Refresh the index.
+     *
+     * @param index index
+     */
+    void refreshIndex(String index);
+
+    /**
+     * Flush the index. The cluster clears cache and completes indexing.
+     *
+     * @param index index
+     */
+    void flushIndex(String index);
+
+}

elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java

@@ -0,0 +1,7 @@
+package org.xbib.elx.api;
+
+@FunctionalInterface
+public interface BulkClientProvider<C extends BulkClient> {
+
+    C getClient();
+}

elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java

@@ -1,487 +0,0 @@
-package org.xbib.elx.api;
-
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Interface for extended managing and indexing methods of an Elasticsearch client.
- */
-public interface ExtendedClient extends Flushable, Closeable {
-
-    /**
-     * Set an Elasticsearch client to extend from it. May be null for TransportClient.
-     * @param client client
-     * @return this client
-     */
-    ExtendedClient setClient(ElasticsearchClient client);
-
-    /**
-     * Return Elasticsearch client.
-     *
-     * @return Elasticsearch client
-     */
-    ElasticsearchClient getClient();
-
-    /**
-     * Get buulk control.
-     * @return the bulk control
-     */
-    BulkController getBulkController();
-
-    /**
-     * Initiative the extended client, the bulk metric and bulk controller,
-     * creates instances and connect to cluster, if required.
-     *
-     * @param settings settings
-     * @return this client
-     * @throws IOException if init fails
-     */
-    ExtendedClient init(Settings settings) throws IOException;
-
-    /**
-     * Build index definition from settings.
-     *
-     * @param index the index name
-     * @param settings the settings for the index
-     * @return index definition
-     * @throws IOException if settings/mapping URL is invalid/malformed
-     */
-    IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
-
-    /**
-     * Add index request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when limits are exceeded.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param create true if document must be created
-     * @param source the source
-     * @return this
-     */
-    ExtendedClient index(String index, String id, boolean create, BytesReference source);
-
-    /**
-     * Index request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when limits are exceeded.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param create true if document is to be created, false otherwise
-     * @param source the source
-     * @return this client methods
-     */
-    ExtendedClient index(String index, String id, boolean create, String source);
-
-    /**
-     * Index request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     *
-     * @param indexRequest the index request to add
-     * @return this
-     */
-    ExtendedClient index(IndexRequest indexRequest);
-
-    /**
-     * Delete request.
-     *
-     * @param index the index
-     * @param id    the id
-     * @return this
-     */
-    ExtendedClient delete(String index, String id);
-
-    /**
-     * Delete request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     *
-     * @param deleteRequest the delete request to add
-     * @return this
-     */
-    ExtendedClient delete(DeleteRequest deleteRequest);
-
-    /**
-     * Bulked update request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     * Note that updates only work correctly when all operations between nodes are synchronized.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param source the source
-     * @return this
-     * @throws IOException if update fails
-     */
-    ExtendedClient update(String index, String id, BytesReference source) throws IOException;
-
-    /**
-     * Update document. Use with precaution! Does not work in all cases.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param source the source
-     * @return this
-     */
-    ExtendedClient update(String index, String id, String source);
-
-    /**
-     * Bulked update request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     * Note that updates only work correctly when all operations between nodes are synchronized.
-     *
-     * @param updateRequest the update request to add
-     * @return this
-     */
-    ExtendedClient update(UpdateRequest updateRequest);
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @return this
-     * @throws IOException if new index creation fails
-     */
-    ExtendedClient newIndex(String index) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @return this
-     * @throws IOException if settings is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     * @param indexDefinition the index definition
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
-
-    /**
-     * Delete an index.
-     * @param indexDefinition the index definition
-     * @return this
-     */
-    ExtendedClient deleteIndex(IndexDefinition indexDefinition);
-
-    /**
-     * Delete an index.
-     *
-     * @param index index
-     * @return this
-     */
-    ExtendedClient deleteIndex(String index);
-
-    /**
-     * Start bulk mode for indexes.
-     * @param indexDefinition index definition
-     * @return this
-     * @throws IOException if bulk could not be started
-     */
-    ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
-
-    /**
-     * Start bulk mode.
-     *
-     * @param index index
-     * @param startRefreshIntervalSeconds refresh interval before bulk
-     * @param stopRefreshIntervalSeconds  refresh interval after bulk
-     * @return this
-     * @throws IOException if bulk could not be started
-     */
-    ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
-                             long stopRefreshIntervalSeconds) throws IOException;
-
-    /**
-     * Stop bulk mode.
-     *
-     * @param indexDefinition index definition
-     * @return this
-     * @throws IOException if bulk could not be startet
-     */
-    ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
-
-    /**
-     * Stops bulk mode.
-     *
-     * @param index index
-     * @param timeout maximum wait time
-     * @param timeUnit time unit for timeout
-     * @return this
-     * @throws IOException if bulk could not be stopped
-     */
-    ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
-
-    /**
-     * Update replica level.
-     * @param indexDefinition the index definition
-     * @param level the replica level
-     * @return this
-     * @throws IOException if replica setting could not be updated
-     */
-    ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
-
-    /**
-     * Update replica level.
-     *
-     * @param index index
-     * @param level the replica level
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return this
-     * @throws IOException if replica setting could not be updated
-     */
-    ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
-
-    /**
-     * Get replica level.
-     * @param indexDefinition the index name
-     * @return the replica level of the index
-     */
-    int getReplicaLevel(IndexDefinition indexDefinition);
-
-    /**
-     * Get replica level.
-     * @param index the index name
-     * @return the replica level of the index
-     */
-    int getReplicaLevel(String index);
-
-    /**
-     * Refresh the index.
-     *
-     * @param index index
-     * @return this
-     */
-    ExtendedClient refreshIndex(String index);
-
-    /**
-     * Flush the index. The cluster clears cache and completes indexing.
-     *
-     * @param index index
-     * @return this
-     */
-    ExtendedClient flushIndex(String index);
-
-    /**
-     * Force segment merge of an index.
-     * @param indexDefinition the index definition
-     * @return this
-     */
-    boolean forceMerge(IndexDefinition indexDefinition);
-
-    /**
-     * Force segment merge of an index.
-     * @param index the index
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return this
-     */
-    boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Wait for all outstanding bulk responses.
-     *
-     * @param timeout maximum wait time
-     * @param timeUnit unit of timeout value
-     * @return true if wait succeeded, false if wait timed out
-     */
-    boolean waitForResponses(long timeout, TimeUnit timeUnit);
-
-    /**
-     * Wait for cluster being healthy.
-     *
-     * @param healthColor cluster health color to wait for
-     * @param maxWaitTime   time value
-     * @param timeUnit time unit
-     * @return true if wait succeeded, false if wait timed out
-     */
-    boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Get current health color.
-     *
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return the cluster health color
-     */
-    String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Wait for index recovery (after replica change).
-     *
-     * @param index index
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return true if wait succeeded, false if wait timed out
-     */
-    boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Update index setting.
-     * @param index the index
-     * @param key the key of the value to be updated
-     * @param value the new value
-     * @param timeout timeout
-     * @param timeUnit time unit
-     * @throws IOException if update index setting failed
-     */
-    void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
-
-    /**
-     * Resolve alias.
-     *
-     * @param alias the alias
-     * @return this index name behind the alias or the alias if there is no index
-     */
-    String resolveAlias(String alias);
-
-    /**
-     * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
-     * name.
-     *
-     * @param alias the alias
-     * @return the most recent index name pointing to the alias
-     */
-    String resolveMostRecentIndex(String alias);
-
-    /**
-     * Get all index aliases.
-     * @param index the index
-     * @return map of index aliases
-     */
-    Map<String, String> getAliases(String index);
-
-    /**
-     *  Shift from one index to another.
-     * @param indexDefinition the index definition
-     * @param additionalAliases new aliases
-     * @return this
-     */
-    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
-
-    /**
-     * Shift from one index to another.
-     * @param indexDefinition the index definition
-     * @param additionalAliases new aliases
-     * @param indexAliasAdder method to add aliases
-     * @return this
-     */
-    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
-                                IndexAliasAdder indexAliasAdder);
-
-    /**
-     * Shift from one index to another.
-     * @param index         the index name
-     * @param fullIndexName the index name with timestamp
-     * @param additionalAliases  a list of names that should be set as index aliases
-     * @return this
-     */
-    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
-
-    /**
-     * Shift from one index to another.
-     * @param index         the index name
-     * @param fullIndexName the index name with timestamp
-     * @param additionalAliases  a list of names that should be set as index aliases
-     * @param adder         an adder method to create alias term queries
-     * @return this
-     */
-    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
-                                IndexAliasAdder adder);
-
-    /**
-     * Prune index.
-     * @param indexDefinition the index definition
-     * @return the index prune result
-     */
-    IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
-
-    /**
-     * Apply retention policy to prune indices. All indices before delta should be deleted,
-     * but the number of mintokeep indices must be kept.
-     *
-     * @param index         index name
-     * @param fullIndexName index name with timestamp
-     * @param delta timestamp delta (for index timestamps)
-     * @param mintokeep     minimum number of indices to keep
-     * @param perform true if pruning should be executed, false if not
-     * @return the index prune result
-     */
-    IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
-
-    /**
-     * Find the timestamp of the most recently indexed document in the index.
-     *
-     * @param index the index name
-     * @param timestampfieldname the timestamp field name
-     * @return millis UTC millis of the most recent document
-     * @throws IOException if most rcent document can not be found
-     */
-    Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
-
-    /**
-     * Get cluster name.
-     * @return the cluster name
-     */
-    String getClusterName();
-}

elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java

@@ -1,7 +0,0 @@
-package org.xbib.elx.api;
-
-@FunctionalInterface
-public interface ExtendedClientProvider<C extends ExtendedClient> {
-
-    C getExtendedClient();
-}

elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java

@@ -1,7 +1,5 @@
 package org.xbib.elx.api;
 
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.concurrent.TimeUnit;
 
 public interface IndexDefinition {
@@ -14,17 +12,13 @@ public interface IndexDefinition {
 
     String getFullIndexName();
 
-    IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException;
+    IndexDefinition setSettings(String settings);
 
-    IndexDefinition setSettingsUrl(URL settingsUrl);
+    String getSettings();
 
-    URL getSettingsUrl();
+    IndexDefinition setMappings(String mappings);
 
-    IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException;
-
-    IndexDefinition setMappingsUrl(URL mappingsUrl);
-
-    URL getMappingsUrl();
+    String getMappings();
 
     IndexDefinition setDateTimePattern(String timeWindow);
 

elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java

@@ -1,6 +1,6 @@
 package org.xbib.elx.api;
 
-import java.util.List;
+import java.util.Collection;
 
 public interface IndexPruneResult {
 
@@ -8,9 +8,9 @@ public interface IndexPruneResult {
 
     State getState();
 
-    List<String> getCandidateIndices();
+    Collection<String> getCandidateIndices();
 
-    List<String> getDeletedIndices();
+    Collection<String> getDeletedIndices();
 
     boolean isAcknowledged();
 }

elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java

@@ -1,10 +1,10 @@
 package org.xbib.elx.api;
 
-import java.util.List;
+import java.util.Collection;
 
 public interface IndexShiftResult {
 
-    List<String> getMovedAliases();
+    Collection<String> getMovedAliases();
 
-    List<String> getNewAliases();
+    Collection<String> getNewAliases();
 }

elx-api/src/main/java/org/xbib/elx/api/NativeClient.java

@@ -0,0 +1,63 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public interface NativeClient extends Closeable {
+
+    /**
+     * Set an Elasticsearch client to extend from it. May be null for TransportClient.
+     * @param client the Elasticsearch client
+     */
+    void setClient(ElasticsearchClient client);
+
+    /**
+     * Return Elasticsearch client.
+     *
+     * @return Elasticsearch client
+     */
+    ElasticsearchClient getClient();
+
+    /**
+     * Initiative the extended client, the bulk metric and bulk controller,
+     * creates instances and connect to cluster, if required.
+     *
+     * @param settings settings
+     * @throws IOException if init fails
+     */
+    void init(Settings settings) throws IOException;
+
+    /**
+     * Get cluster name.
+     * @return the cluster name
+     */
+    String getClusterName();
+
+    /**
+     * Get current health color.
+     *
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return the cluster health color
+     */
+    String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
+
+    /**
+     * Wait for cluster being healthy.
+     *
+     * @param healthColor cluster health color to wait for
+     * @param maxWaitTime   time value
+     * @param timeUnit time unit
+     */
+    void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
+
+    Map<String, ?> getMapping(String index, String mapping);
+
+    long getSearchableDocs(String index);
+
+    boolean isIndexExists(String index);
+}

elx-api/src/main/java/org/xbib/elx/api/ReadClient.java

@@ -1,25 +1,27 @@
 package org.xbib.elx.api;
 
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetRequestBuilder;
 import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetRequest;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
 import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
 
-public interface ReadClient {
+public interface SearchClient {
 
-    ActionFuture<GetResponse> get(GetRequest getRequest);
+    Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
 
-    void get(GetRequest request, ActionListener<GetResponse> listener);
+    Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
 
-    ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
+    Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
 
-    void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
+    Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
+                             TimeValue scrollTime, int scrollSize);
 
-    ActionFuture<SearchResponse> search(SearchRequest request);
-
-    void search(SearchRequest request, ActionListener<SearchResponse> listener);
+    Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
 }

elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java

@@ -0,0 +1,7 @@
+package org.xbib.elx.api;
+
+@FunctionalInterface
+public interface SearchClientProvider<C extends SearchClient> {
+
+    C getClient();
+}

elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java

@@ -2,10 +2,8 @@ package org.xbib.elx.common;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchTimeoutException;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
@@ -18,14 +16,9 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
-import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
-import org.elasticsearch.action.admin.indices.flush.FlushAction;
-import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexAction;
@@ -34,33 +27,25 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
-import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -69,6 +54,7 @@ import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
 import org.xbib.elx.api.BulkController;
+import org.xbib.elx.api.BulkMetric;
 import org.xbib.elx.api.ExtendedClient;
 import org.xbib.elx.api.IndexAliasAdder;
 import org.xbib.elx.api.IndexDefinition;
@@ -78,13 +64,14 @@ import org.xbib.elx.api.IndexShiftResult;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.charset.MalformedInputException;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -97,22 +84,32 @@ import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-public abstract class AbstractExtendedClient implements ExtendedClient {
+public abstract class AbstractAdminClient extends AbstractNativeClient implements AdminClient {
 
     private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName());
 
+    /**
+     * The one and only index type name used in the extended client.
+     * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
+     */
+    private static final String TYPE_NAME = "doc";
+
+    /**
+     * The Elasticsearch client.
+     */
     private ElasticsearchClient client;
 
+    private BulkMetric bulkMetric;
+
     private BulkController bulkController;
 
-    private final AtomicBoolean closed;
+    private AtomicBoolean closed;
 
     private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
         @Override
@@ -148,14 +145,6 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         }
     };
 
-    protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
-
-    protected abstract void closeClient() throws IOException;
-
-    protected AbstractExtendedClient() {
-        closed = new AtomicBoolean(false);
-    }
-
     @Override
     public AbstractExtendedClient setClient(ElasticsearchClient client) {
         this.client = client;
@@ -167,6 +156,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         return client;
     }
 
+    @Override
+    public BulkMetric getBulkMetric() {
+        return bulkMetric;
+    }
+
     @Override
     public BulkController getBulkController() {
         return bulkController;
@@ -174,12 +168,16 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public AbstractExtendedClient init(Settings settings) throws IOException {
-        logger.info("initializing with settings = " + settings.toDelimitedString(','));
+        logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
         if (client == null) {
             client = createClient(settings);
         }
+        if (bulkMetric == null) {
+            bulkMetric = new DefaultBulkMetric();
+            bulkMetric.init(settings);
+        }
         if (bulkController == null) {
-            this.bulkController = new DefaultBulkController(this);
+            bulkController = new DefaultBulkController(this, bulkMetric);
             bulkController.init(settings);
         }
         return this;
@@ -194,8 +192,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public void close() throws IOException {
-        ensureClient();
+        ensureActive();
         if (closed.compareAndSet(false, true)) {
+            if (bulkMetric != null) {
+                logger.info("closing bulk metric");
+                bulkMetric.close();
+            }
             if (bulkController != null) {
                 logger.info("closing bulk controller");
                 bulkController.close();
@@ -206,7 +208,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public String getClusterName() {
-        ensureClient();
+        ensureActive();
         try {
             ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
             ClusterStateResponse clusterStateResponse =
@@ -226,7 +228,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException {
-        ensureClient();
+        ensureActive();
         waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
         URL indexSettings = indexDefinition.getSettingsUrl();
         if (indexSettings == null) {
@@ -261,30 +263,32 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public ExtendedClient newIndex(String index) throws IOException {
-        return newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
+        return newIndex(index, Settings.EMPTY, (Map<String, Object>) null);
     }
 
     @Override
     public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException {
         return newIndex(index,
-                Settings.settingsBuilder().loadFromStream(".json", settings).build(),
-                mapping != null ? JsonXContent.jsonXContent.createParser(mapping).mapOrdered() : null);
+                Settings.builder().loadFromStream(".json", settings, true).build(),
+                mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
     }
 
     @Override
     public ExtendedClient newIndex(String index, Settings settings) throws IOException {
-        return newIndex(index, settings, (Map<String, ?>) null);
+        return newIndex(index, settings, (Map<String, Object>) null);
     }
 
     @Override
     public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException {
         return newIndex(index, settings,
-                mapping != null ? JsonXContent.jsonXContent.createParser(mapping).mapOrdered() : null);
+                mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
     }
 
     @Override
-    public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) {
-        ensureClient();
+    public ExtendedClient newIndex(String index, Settings settings, Map<String, Object> mapping) throws IOException {
+        ensureActive();
         if (index == null) {
             logger.warn("no index name given to create index");
             return this;
@@ -294,35 +298,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
             createIndexRequest.settings(settings);
         }
         if (mapping != null) {
-            createIndexRequest.mapping("doc", mapping);
+            createIndexRequest.mapping(TYPE_NAME, mapping);
         }
         CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
-        if (createIndexResponse.isAcknowledged()) {
-            return this;
-        }
-        throw new IllegalStateException("index creation not acknowledged: " + index);
-    }
-
-
-    @Override
-    public ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) {
-        ensureClient();
-        if (index == null) {
-            logger.warn("no index name given to create index");
-            return this;
-        }
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
-        if (settings != null) {
-            createIndexRequest.settings(settings);
-        }
-        if (mapping != null) {
-            createIndexRequest.mapping("doc", mapping);
-        }
-        CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
-        if (createIndexResponse.isAcknowledged()) {
-            return this;
-        }
-        throw new IllegalStateException("index creation not acknowledged: " + index);
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        logger.info("index {} created: {}", index,
+                Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
+        return this;
     }
 
     @Override
@@ -332,7 +314,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public ExtendedClient deleteIndex(String index) {
-        ensureClient();
+        ensureActive();
         if (index == null) {
             logger.warn("no index name given to delete index");
             return this;
@@ -352,7 +334,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
             throws IOException {
         if (bulkController != null) {
-            ensureClient();
+            ensureActive();
             bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
         }
         return this;
@@ -361,7 +343,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException {
         if (bulkController != null) {
-            ensureClient();
+            ensureActive();
             bulkController.stopBulkMode(indexDefinition);
         }
         return this;
@@ -370,7 +352,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
         if (bulkController != null) {
-            ensureClient();
+            ensureActive();
             bulkController.stopBulkMode(index, timeout, timeUnit);
         }
         return this;
@@ -378,63 +360,63 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public ExtendedClient index(String index, String id, boolean create, String source) {
-        return index(new IndexRequest().index(index).type("doc").id(id).create(create)
-                .source(source.getBytes(StandardCharsets.UTF_8)));
+        return index(new IndexRequest(index, TYPE_NAME, id).create(create)
+                .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
     }
 
     @Override
     public ExtendedClient index(String index, String id, boolean create, BytesReference source) {
-        return index(new IndexRequest().index(index).type("doc").id(id).create(create)
-                .source(source));
+        return index(new IndexRequest(index, TYPE_NAME, id).create(create)
+                .source(source, XContentType.JSON));
     }
 
     @Override
     public ExtendedClient index(IndexRequest indexRequest) {
-        ensureClient();
-        bulkController.bulkIndex(indexRequest);
+        ensureActive();
+        bulkController.index(indexRequest);
         return this;
     }
 
     @Override
     public ExtendedClient delete(String index, String id) {
-        return delete(new DeleteRequest().index(index).type("doc").id(id));
+        return delete(new DeleteRequest(index, TYPE_NAME, id));
     }
 
     @Override
     public ExtendedClient delete(DeleteRequest deleteRequest) {
-        ensureClient();
-        bulkController.bulkDelete(deleteRequest);
+        ensureActive();
+        bulkController.delete(deleteRequest);
         return this;
     }
 
     @Override
     public ExtendedClient update(String index, String id, BytesReference source) {
-        return update(new UpdateRequest().index(index).type("doc").id(id)
-                .doc(source));
+        return update(new UpdateRequest(index, TYPE_NAME, id)
+                .doc(source, XContentType.JSON));
     }
 
     @Override
     public ExtendedClient update(String index, String id, String source) {
-        return update(new UpdateRequest().index(index).type("doc").id(id)
-                .doc(source.getBytes(StandardCharsets.UTF_8)));
+        return update(new UpdateRequest(index, TYPE_NAME, id)
+                .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
     }
 
     @Override
     public ExtendedClient update(UpdateRequest updateRequest) {
-        ensureClient();
-        bulkController.bulkUpdate(updateRequest);
+        ensureActive();
+        bulkController.update(updateRequest);
         return this;
     }
 
     @Override
     public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
-        ensureClient();
-        return bulkController.waitForBulkResponses(timeout, timeUnit);
+        ensureActive();
+        return bulkController.waitForResponses(timeout, timeUnit);
     }
 
     @Override
     public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
-        ensureClient();
+        ensureActive();
         ensureIndexGiven(index);
         GetSettingsRequest settingsRequest = new GetSettingsRequest();
         settingsRequest.indices(index);
@@ -443,13 +425,13 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         if (shards > 0) {
             TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
             ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
-                    .indices(new String[]{index})
+                    .indices(index)
                     .waitForActiveShards(shards)
                     .timeout(timeout);
             ClusterHealthResponse healthResponse =
                     client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
             if (healthResponse != null && healthResponse.isTimedOut()) {
-                logger.warn("timeout waiting for recovery");
+                logger.error("timeout waiting for recovery");
                 return false;
             }
         }
@@ -458,13 +440,15 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
-        ensureClient();
+        ensureActive();
         ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
         TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
         ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
                 new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
         if (healthResponse != null && healthResponse.isTimedOut()) {
-            logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
+            if (logger.isErrorEnabled()) {
+                logger.error("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
+            }
             return false;
         }
         return true;
@@ -472,7 +456,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
-        ensureClient();
+        ensureActive();
         try {
             TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
             ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
@@ -498,7 +482,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     }
 
     @Override
-    public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
+    public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
         waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations
         if (level > 0) {
             updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
@@ -529,7 +513,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public ExtendedClient flushIndex(String index) {
         if (index != null) {
-            ensureClient();
+            ensureActive();
             client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
         }
         return this;
@@ -538,7 +522,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public ExtendedClient refreshIndex(String index) {
         if (index != null) {
-            ensureClient();
+            ensureActive();
             client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
         }
         return this;
@@ -546,10 +530,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public String resolveMostRecentIndex(String alias) {
+        ensureActive();
         if (alias == null) {
             return null;
         }
-        ensureClient();
         GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias);
         GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
         Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
@@ -563,6 +547,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         return indices.isEmpty() ? alias : indices.iterator().next();
     }
 
+    @Override
     public Map<String, String> getAliases(String index) {
         if (index == null) {
             return Collections.emptyMap();
@@ -573,7 +558,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public String resolveAlias(String alias) {
-        ensureClient();
+        ensureActive();
         ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.blocks(false);
         clusterStateRequest.metaData(true);
@@ -584,7 +569,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
         SortedMap<String, AliasOrIndex> map = clusterStateResponse.getState().getMetaData().getAliasAndIndexLookup();
         AliasOrIndex aliasOrIndex = map.get(alias);
-        return aliasOrIndex != null ? aliasOrIndex.getIndices().iterator().next().getIndex() : null;
+        return aliasOrIndex != null ? aliasOrIndex.getIndices().iterator().next().getIndex().getName() : null;
     }
 
     @Override
@@ -615,7 +600,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public IndexShiftResult shiftIndex(String index, String fullIndexName,
                                      List<String> additionalAliases, IndexAliasAdder adder) {
-        ensureClient();
+        ensureActive();
         if (index == null) {
             return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
         }
@@ -631,8 +616,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         final List<String> moveAliases = new ArrayList<>();
         IndicesAliasesRequest indicesAliasesRequest = new IndicesAliasesRequest();
         if (oldAliasMap == null || !oldAliasMap.containsKey(index)) {
-            indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                    fullIndexName, index));
+            indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                    .index(fullIndexName).alias(index));
             newAliases.add(index);
         }
         // move existing aliases
@@ -640,14 +625,14 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
             for (Map.Entry<String, String> entry : oldAliasMap.entrySet()) {
                 String alias = entry.getKey();
                 String filter = entry.getValue();
-                indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.REMOVE,
-                        oldIndex, alias));
+                indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
+                        .indices(oldIndex).alias(alias));
                 if (filter != null) {
-                    indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                            fullIndexName, alias).filter(filter));
+                    indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                            .index(fullIndexName).alias(alias).filter(filter));
                 } else {
-                    indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                            fullIndexName, alias));
+                    indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                            .index(fullIndexName).alias(alias));
                 }
                 moveAliases.add(alias);
             }
@@ -660,37 +645,31 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                     if (adder != null) {
                         adder.addIndexAlias(indicesAliasesRequest, fullIndexName, additionalAlias);
                     } else {
-                        indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                                fullIndexName, additionalAlias));
+                        indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                                .index(fullIndexName).alias(additionalAlias));
                     }
                     newAliases.add(additionalAlias);
                 } else {
                     String filter = oldAliasMap.get(additionalAlias);
-                    indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.REMOVE,
-                            oldIndex, additionalAlias));
+                    indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
+                            .indices(oldIndex).alias(additionalAlias));
                     if (filter != null) {
-                        indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                                fullIndexName, additionalAlias).filter(filter));
+                        indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                                .index(fullIndexName).alias(additionalAlias).filter(filter));
                     } else {
-                        indicesAliasesRequest.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                                fullIndexName, additionalAlias));
+                        indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                                .index(fullIndexName).alias(additionalAlias));
                     }
                     moveAliases.add(additionalAlias);
                 }
             }
         }
         if (!indicesAliasesRequest.getAliasActions().isEmpty()) {
-            StringBuilder sb = new StringBuilder();
-            for (IndicesAliasesRequest.AliasActions aliasActions : indicesAliasesRequest.getAliasActions()) {
-                sb.append("[").append(aliasActions.actionType().name())
-                        .append(",indices=").append(Arrays.asList(aliasActions.indices()))
-                        .append(",aliases=").append(Arrays.asList(aliasActions.aliases())).append("]");
-            }
-            logger.debug("indices alias request = {}", sb.toString());
+            logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString());
             IndicesAliasesResponse indicesAliasesResponse =
                     client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
-            logger.debug("response isAcknowledged = {}",
-                    indicesAliasesResponse.isAcknowledged());
+            logger.debug("response isAcknowledged = {} isFragment = {}",
+                    indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment());
         }
         return new SuccessIndexShiftResult(moveAliases, newAliases);
     }
@@ -709,11 +688,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         if (index.equals(fullIndexName)) {
             return EMPTY_INDEX_PRUNE_RESULT;
         }
-        ensureClient();
+        ensureActive();
         GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
         GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
         Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
-        logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length);
+        logger.info("{} indices", getIndexResponse.getIndices().length);
         List<String> candidateIndices = new ArrayList<>();
         for (String s : getIndexResponse.getIndices()) {
             Matcher m = pattern.matcher(s);
@@ -749,30 +728,22 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest()
                 .indices(indicesToDelete.toArray(s));
         DeleteIndexResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
-        if (response.isAcknowledged()) {
-            logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s));
-            waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
-            return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
-        } else {
-            logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s));
-            return new FailPruneResult(candidateIndices, indicesToDelete, response);
-        }
+        return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
     }
 
     @Override
     public Long mostRecentDocument(String index, String timestampfieldname) {
-        ensureClient();
-        SortBuilder sort = SortBuilders
-                .fieldSort(timestampfieldname)
-                .order(SortOrder.DESC);
-        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
-                .sort(sort)
-                .field(timestampfieldname)
-                .size(1);
-        SearchRequest searchRequest = new SearchRequest()
-                .indices(index)
-                .source(sourceBuilder);
-        SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
+        ensureActive();
+        SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
+        SearchSourceBuilder builder = new SearchSourceBuilder();
+        builder.sort(sort);
+        builder.storedField(timestampfieldname);
+        builder.size(1);
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.indices(index);
+        searchRequest.source(builder);
+        SearchResponse searchResponse =
+                client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
         if (searchResponse.getHits().getHits().length == 1) {
             SearchHit hit = searchResponse.getHits().getHits()[0];
             if (hit.getFields().get(timestampfieldname) != null) {
@@ -815,7 +786,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
             throws IOException {
-        boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient));
+        boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockAdminClient));
         String indexName = settings.get("name", index);
         String fullIndexName;
         String dateTimePattern = settings.get("dateTimePattern");
@@ -835,8 +806,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 .setEnabled(isEnabled)
                 .setIndex(indexName)
                 .setFullIndexName(fullIndexName)
-                .setSettingsUrl(settings.get("settings"))
-                .setMappingsUrl(settings.get("mapping"))
+                .setSettings(findSettingsFrom(settings.get("settings")))
+                .setMappings(findMappingsFrom(settings.get("mapping")))
                 .setDateTimePattern(dateTimePattern)
                 .setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
                 .setShift(settings.getAsBoolean("shift", true))
@@ -849,29 +820,56 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
-        ensureClient();
+        ensureActive();
         if (index == null) {
             throw new IOException("no index name given");
         }
-        if (key == null) {
-            throw new IOException("no key given");
+        try {
+            URL url = new URL(string);
+            try (InputStream inputStream = url.openStream()) {
+                Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build();
+                XContentBuilder builder = JsonXContent.contentBuilder();
+                settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                return Strings.toString(builder);
+            }
+        } catch (MalformedURLException e) {
+            return string;
         }
         if (value == null) {
             throw new IOException("no value given");
         }
-        Settings.Builder updateSettingsBuilder = Settings.settingsBuilder();
+        Settings.Builder updateSettingsBuilder = Settings.builder();
         updateSettingsBuilder.put(key, value.toString());
         UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
                 .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
         client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
     }
 
-    private void ensureClient() {
+    private void ensureActive() {
         if (this instanceof MockExtendedClient) {
             return;
         }
-        if (client == null) {
-            throw new IllegalStateException("no client");
+        try {
+            URL url = new URL(string);
+            try (InputStream inputStream = url.openStream()) {
+                if (string.endsWith(".json")) {
+                    Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                                    DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
+                    XContentBuilder builder = JsonXContent.contentBuilder();
+                    builder.startObject().map(mappings).endObject();
+                    return Strings.toString(builder);
+                }
+                if (string.endsWith(".yml") || string.endsWith(".yaml")) {
+                    Map<String, ?> mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY,
+                            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
+                    XContentBuilder builder = JsonXContent.contentBuilder();
+                    builder.startObject().map(mappings).endObject();
+                    return Strings.toString(builder);
+                }
+            }
+            return string;
+        } catch (MalformedInputException e) {
+            return string;
         }
     }
 
@@ -898,7 +896,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     }
 
     public void checkMapping(String index) {
-        ensureClient();
+        ensureActive();
         GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
         GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
         ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
@@ -914,24 +912,27 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     private void checkMapping(String index, String type, MappingMetaData mappingMetaData) {
         try {
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices(index)
-                    .source(builder);
+            SearchSourceBuilder builder = new SearchSourceBuilder();
+            builder.query(QueryBuilders.matchAllQuery());
+            builder.size(0);
+            SearchRequest searchRequest = new SearchRequest();
+            searchRequest.indices(index);
+            searchRequest.types(type);
+            searchRequest.source(builder);
             SearchResponse searchResponse =
                     client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
             long total = searchResponse.getHits().getTotalHits();
             if (total > 0L) {
                 Map<String, Long> fields = new TreeMap<>();
                 Map<String, Object> root = mappingMetaData.getSourceAsMap();
-                checkMapping(index, "", "", root, fields);
+                checkMapping(index, type, "", "", root, fields);
                 AtomicInteger empty = new AtomicInteger();
                 Map<String, Long> map = sortByValue(fields);
                 map.forEach((key, value) -> {
                     logger.info("{} {} {}",
-                            key, value, (double) value * 100 / total);
+                            key,
+                            value,
+                            (double) value * 100 / total);
                     if (value == 0) {
                         empty.incrementAndGet();
                     }
@@ -945,7 +946,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     }
 
     @SuppressWarnings("unchecked")
-    private void checkMapping(String index,
+    private void checkMapping(String index, String type,
                               String pathDef, String fieldName, Map<String, Object> map,
                               Map<String, Long> fields) {
         String path = pathDef;
@@ -970,17 +971,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 String fieldType = o instanceof String ? o.toString() : null;
                 // do not recurse into our custom field mapper
                 if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) {
-                    checkMapping(index, path, key, child, fields);
+                    checkMapping(index, type, path, key, child, fields);
                 }
             } else if ("type".equals(key)) {
                 QueryBuilder filterBuilder = QueryBuilders.existsQuery(path);
                 QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder);
-                SearchSourceBuilder builder = new SearchSourceBuilder()
-                        .query(queryBuilder)
-                        .size(0);
-                SearchRequest searchRequest = new SearchRequest()
-                        .indices(index)
-                        .source(builder);
+                SearchSourceBuilder builder = new SearchSourceBuilder();
+                builder.query(queryBuilder);
+                builder.size(0);
+                SearchRequest searchRequest = new SearchRequest();
+                searchRequest.indices(index);
+                searchRequest.types(type);
+                searchRequest.source(builder);
                 SearchResponse searchResponse =
                         client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
                 fields.put(path, searchResponse.getHits().getTotalHits());
@@ -995,58 +997,38 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         return result;
     }
 
-    private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
-        switch (timeUnit) {
-            case DAYS:
-                return TimeValue.timeValueHours(24 * timeValue);
-            case HOURS:
-                return TimeValue.timeValueHours(timeValue);
-            case MINUTES:
-                return TimeValue.timeValueMinutes(timeValue);
-            case SECONDS:
-                return TimeValue.timeValueSeconds(timeValue);
-            case MILLISECONDS:
-                return TimeValue.timeValueMillis(timeValue);
-            case MICROSECONDS:
-                return TimeValue.timeValueNanos(1000 * timeValue);
-            case NANOSECONDS:
-                return TimeValue.timeValueNanos(timeValue);
-            default:
-                throw new IllegalArgumentException("unknown time unit: " + timeUnit);
-        }
-    }
-
     private static class SuccessIndexShiftResult implements IndexShiftResult {
 
-        List<String> movedAliases;
+        Collection<String> movedAliases;
 
-        List<String> newAliases;
+        Collection<String> newAliases;
 
-        SuccessIndexShiftResult(List<String> movedAliases, List<String> newAliases) {
+        SuccessIndexShiftResult(Collection<String> movedAliases, Collection<String> newAliases) {
             this.movedAliases = movedAliases;
             this.newAliases = newAliases;
         }
 
         @Override
-        public List<String> getMovedAliases() {
+        public Collection<String> getMovedAliases() {
             return movedAliases;
         }
 
         @Override
-        public List<String> getNewAliases() {
+        public Collection<String> getNewAliases() {
             return newAliases;
         }
     }
 
     private static class SuccessPruneResult implements IndexPruneResult {
 
-        List<String> candidateIndices;
+        Collection<String> candidateIndices;
 
-        List<String> indicesToDelete;
+        Collection<String> indicesToDelete;
 
         DeleteIndexResponse response;
 
-        SuccessPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
+        SuccessPruneResult(Collection<String> candidateIndices,
+                           Collection<String> indicesToDelete,
                            DeleteIndexResponse response) {
             this.candidateIndices = candidateIndices;
             this.indicesToDelete = indicesToDelete;
@@ -1059,48 +1041,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         }
 
         @Override
-        public List<String> getCandidateIndices() {
-            return candidateIndices;
-        }
-
-        @Override
-        public List<String> getDeletedIndices() {
-            return indicesToDelete;
-        }
-
-        @Override
-        public boolean isAcknowledged() {
-            return response.isAcknowledged();
-        }
-    }
-
-    private static class FailPruneResult implements IndexPruneResult {
-
-        List<String> candidateIndices;
-
-        List<String> indicesToDelete;
-
-        DeleteIndexResponse response;
-
-        FailPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
-                           DeleteIndexResponse response) {
-            this.candidateIndices = candidateIndices;
-            this.indicesToDelete = indicesToDelete;
-            this.response = response;
-        }
-
-        @Override
-        public IndexPruneResult.State getState() {
-            return IndexPruneResult.State.FAIL;
-        }
-
-        @Override
-        public List<String> getCandidateIndices() {
+        public Collection<String> getCandidateIndices() {
             return candidateIndices;
         }
 
         @Override
-        public List<String> getDeletedIndices() {
+        public Collection<String> getDeletedIndices() {
             return indicesToDelete;
         }
 
@@ -1112,11 +1058,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     private static class NothingToDoPruneResult implements IndexPruneResult {
 
-        List<String> candidateIndices;
+        Collection<String> candidateIndices;
 
-        List<String> indicesToDelete;
+        Collection<String> indicesToDelete;
 
-        NothingToDoPruneResult(List<String> candidateIndices, List<String> indicesToDelete) {
+        NothingToDoPruneResult(Collection<String> candidateIndices, List<String> indicesToDelete) {
             this.candidateIndices = candidateIndices;
             this.indicesToDelete = indicesToDelete;
         }
@@ -1127,12 +1073,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         }
 
         @Override
-        public List<String> getCandidateIndices() {
+        public Collection<String> getCandidateIndices() {
             return candidateIndices;
         }
 
         @Override
-        public List<String> getDeletedIndices() {
+        public Collection<String> getDeletedIndices() {
             return indicesToDelete;
         }
 

elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java

@@ -0,0 +1,246 @@
+package org.xbib.elx.common;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.flush.FlushAction;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.xbib.elx.api.BulkClient;
+import org.xbib.elx.api.BulkController;
+import org.xbib.elx.api.BulkMetric;
+import org.xbib.elx.api.IndexDefinition;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public abstract class AbstractBulkClient extends AbstractNativeClient implements BulkClient {
+
+    private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
+
+    private BulkMetric bulkMetric;
+
+    private BulkController bulkController;
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
+        super.init(settings);
+        if (bulkMetric == null) {
+            bulkMetric = new DefaultBulkMetric();
+            bulkMetric.init(settings);
+        }
+        if (bulkController == null) {
+            bulkController = new DefaultBulkController(this, bulkMetric);
+            bulkController.init(settings);
+        }
+    }
+
+    @Override
+    public BulkMetric getBulkMetric() {
+        return bulkMetric;
+    }
+
+    @Override
+    public BulkController getBulkController() {
+        return bulkController;
+    }
+
+    @Override
+    public void flush() throws IOException {
+        if (bulkController != null) {
+            bulkController.flush();
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        ensureClientIsPresent();
+        if (closed.compareAndSet(false, true)) {
+            if (bulkMetric != null) {
+                logger.info("closing bulk metric");
+                bulkMetric.close();
+            }
+            if (bulkController != null) {
+                logger.info("closing bulk controller");
+                bulkController.close();
+            }
+            closeClient();
+        }
+    }
+
+    @Override
+    public void newIndex(IndexDefinition indexDefinition) throws IOException {
+        Settings settings = indexDefinition.getSettings() == null ? null :
+                Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
+        Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
+                JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
+        newIndex(indexDefinition.getFullIndexName(), settings, mappings);
+    }
+
+    @Override
+    public void newIndex(String index) throws IOException {
+        newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
+    }
+
+    @Override
+    public void newIndex(String index, Settings settings) throws IOException {
+        newIndex(index, settings, (Map<String, ?>) null);
+    }
+
+    @Override
+    public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
+        String mappingString = Strings.toString(builder);
+        Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
+        newIndex(index, settings, mappings);
+    }
+
+    @Override
+    public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
+        if (index == null) {
+            logger.warn("no index name given to create index");
+            return;
+        }
+        ensureClientIsPresent();
+        waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
+        if (settings != null) {
+            createIndexRequest.settings(settings);
+        }
+        if (mapping != null) {
+            createIndexRequest.mapping(TYPE_NAME, mapping);
+        }
+        CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        logger.info("index {} created: {}", index,
+                Strings.toString(createIndexResponse.toXContent(builder, ToXContent.EMPTY_PARAMS)));
+    }
+
+    @Override
+    public void startBulk(IndexDefinition indexDefinition) throws IOException {
+        startBulk(indexDefinition.getFullIndexName(), -1, 1);
+    }
+
+    @Override
+    public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
+            throws IOException {
+        if (bulkController != null) {
+            ensureClientIsPresent();
+            bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
+        }
+    }
+
+    @Override
+    public void stopBulk(IndexDefinition indexDefinition) throws IOException {
+        if (bulkController != null) {
+            ensureClientIsPresent();
+            bulkController.stopBulkMode(indexDefinition);
+        }
+    }
+
+    @Override
+    public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
+        if (bulkController != null) {
+            ensureClientIsPresent();
+            bulkController.stopBulkMode(index, timeout, timeUnit);
+        }
+    }
+
+    @Override
+    public BulkClient index(String index, String id, boolean create, String source) {
+        return index(new IndexRequest(index, TYPE_NAME, id).create(create)
+                .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient index(String index, String id, boolean create, BytesReference source) {
+        return index(new IndexRequest(index, TYPE_NAME, id).create(create)
+                .source(source, XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient index(IndexRequest indexRequest) {
+        ensureClientIsPresent();
+        bulkController.index(indexRequest);
+        return this;
+    }
+
+    @Override
+    public BulkClient delete(String index, String id) {
+        return delete(new DeleteRequest(index, TYPE_NAME, id));
+    }
+
+    @Override
+    public BulkClient delete(DeleteRequest deleteRequest) {
+        ensureClientIsPresent();
+        bulkController.delete(deleteRequest);
+        return this;
+    }
+
+    @Override
+    public BulkClient update(String index, String id, BytesReference source) {
+        return update(new UpdateRequest(index, TYPE_NAME, id)
+                .doc(source, XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient update(String index, String id, String source) {
+        return update(new UpdateRequest(index, TYPE_NAME, id)
+                .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient update(UpdateRequest updateRequest) {
+        ensureClientIsPresent();
+        bulkController.update(updateRequest);
+        return this;
+    }
+
+    @Override
+    public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        return bulkController.waitForResponses(timeout, timeUnit);
+    }
+
+    @Override
+    public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
+        super.updateIndexSetting(index, key, value, timeout, timeUnit);
+    }
+
+    @Override
+    public void flushIndex(String index) {
+        if (index != null) {
+            ensureClientIsPresent();
+            client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
+        }
+    }
+
+    @Override
+    public void refreshIndex(String index) {
+        if (index != null) {
+            ensureClientIsPresent();
+            client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
+        }
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/AbstractNativeClient.java

@@ -0,0 +1,218 @@
+package org.xbib.elx.common;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.xbib.elx.api.NativeClient;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public abstract class AbstractNativeClient implements NativeClient {
+
+    private static final Logger logger = LogManager.getLogger(AbstractNativeClient.class.getName());
+
+    /**
+     * The one and only index type name used in the extended client.
+     * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
+     */
+    protected static final String TYPE_NAME = "doc";
+
+    protected ElasticsearchClient client;
+
+    protected final AtomicBoolean closed;
+
+    protected AbstractNativeClient() {
+        closed = new AtomicBoolean(false);
+    }
+
+    @Override
+    public void setClient(ElasticsearchClient client) {
+        this.client = client;
+    }
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return client;
+    }
+
+    protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
+
+    protected abstract void closeClient() throws IOException;
+
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
+        if (client == null) {
+            client = createClient(settings);
+        }
+    }
+
+    @Override
+    public String getClusterName() {
+        ensureClientIsPresent();
+        try {
+            ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
+            ClusterStateResponse clusterStateResponse =
+                    getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
+            return clusterStateResponse.getClusterName().value();
+        } catch (ElasticsearchTimeoutException e) {
+            logger.warn(e.getMessage(), e);
+            return "TIMEOUT";
+        } catch (NoNodeAvailableException e) {
+            logger.warn(e.getMessage(), e);
+            return "DISCONNECTED";
+        } catch (Exception e) {
+            logger.warn(e.getMessage(), e);
+            return "[" + e.getMessage() + "]";
+        }
+    }
+
+    @Override
+    public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
+        TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
+        ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
+                new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
+        if (healthResponse != null && healthResponse.isTimedOut()) {
+            String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
+            if (logger.isErrorEnabled()) {
+                logger.error(message);
+            }
+            throw new IllegalStateException(message);
+        }
+    }
+
+    @Override
+    public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        try {
+            TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
+            ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
+                    new ClusterHealthRequest().timeout(timeout)).actionGet();
+            ClusterHealthStatus status = healthResponse.getStatus();
+            return status.name();
+        } catch (ElasticsearchTimeoutException e) {
+            logger.warn(e.getMessage(), e);
+            return "TIMEOUT";
+        } catch (NoNodeAvailableException e) {
+            logger.warn(e.getMessage(), e);
+            return "DISCONNECTED";
+        } catch (Exception e) {
+            logger.warn(e.getMessage(), e);
+            return "[" + e.getMessage() + "]";
+        }
+    }
+
+    @Override
+    public Map<String, ?> getMapping(String index, String mapping) {
+        GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
+                .setIndices(index)
+                .setTypes(mapping);
+        GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
+        logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
+        return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
+    }
+
+    @Override
+    public long getSearchableDocs(String index) {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
+                .setIndices(index)
+                .setQuery(QueryBuilders.matchAllQuery())
+                .setSize(0);
+        return searchRequestBuilder.execute().actionGet().getHits().getTotalHits();
+    }
+
+    @Override
+    public boolean isIndexExists(String index) {
+        IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
+        indicesExistsRequest.indices(index);
+        IndicesExistsResponse indicesExistsResponse =
+                client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
+        return indicesExistsResponse.isExists();
+    }
+
+
+    @Override
+    public void close() throws IOException {
+        ensureClientIsPresent();
+        if (closed.compareAndSet(false, true)) {
+            closeClient();
+        }
+    }
+
+    protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
+        ensureClientIsPresent();
+        if (index == null) {
+            throw new IOException("no index name given");
+        }
+        if (key == null) {
+            throw new IOException("no key given");
+        }
+        if (value == null) {
+            throw new IOException("no value given");
+        }
+        Settings.Builder updateSettingsBuilder = Settings.builder();
+        updateSettingsBuilder.put(key, value.toString());
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
+                .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
+        client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
+    }
+
+    protected void ensureClientIsPresent() {
+        if (this instanceof MockAdminClient) {
+            return;
+        }
+        if (client == null) {
+            throw new IllegalStateException("no client");
+        }
+    }
+
+    protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
+        switch (timeUnit) {
+            case DAYS:
+                return TimeValue.timeValueHours(24 * timeValue);
+            case HOURS:
+                return TimeValue.timeValueHours(timeValue);
+            case MINUTES:
+                return TimeValue.timeValueMinutes(timeValue);
+            case SECONDS:
+                return TimeValue.timeValueSeconds(timeValue);
+            case MILLISECONDS:
+                return TimeValue.timeValueMillis(timeValue);
+            case MICROSECONDS:
+                return TimeValue.timeValueNanos(1000 * timeValue);
+            case NANOSECONDS:
+                return TimeValue.timeValueNanos(timeValue);
+            default:
+                throw new IllegalArgumentException("unknown time unit: " + timeUnit);
+        }
+    }
+
+}

elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java

@@ -0,0 +1,158 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.get.GetAction;
+import org.elasticsearch.action.get.GetRequestBuilder;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetAction;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequestBuilder;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.search.SearchScrollRequestBuilder;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import org.xbib.elx.api.SearchClient;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Optional;
+import java.util.Spliterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+public abstract class AbstractSearchClient extends AbstractNativeClient implements SearchClient {
+
+    @Override
+    public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
+        GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
+        getRequestBuilderConsumer.accept(getRequestBuilder);
+        GetResponse getResponse = getRequestBuilder.execute().actionGet();
+        return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
+    }
+
+    @Override
+    public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
+        MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
+        multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
+        MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
+        return multiGetItemResponse.getResponses().length == 0 ?  Optional.empty() : Optional.of(multiGetItemResponse);
+    }
+
+    @Override
+    public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
+        queryBuilder.accept(searchRequestBuilder);
+        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+        if (searchResponse.getFailedShards() > 0) {
+            StringBuilder sb = new StringBuilder("Search failed:");
+            for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
+                sb.append("\n").append(failure.reason());
+            }
+            throw new ElasticsearchException(sb.toString());
+        }
+        return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
+    }
+
+    @Override
+    public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
+                                         TimeValue scrollTime, int scrollSize) {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
+        queryBuilder.accept(searchRequestBuilder);
+        searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
+        SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
+        Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
+                searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
+                        .setScrollId(searchResponse.getScrollId())
+                        .setScroll(scrollTime)
+                        .execute().actionGet());
+        Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
+        Consumer<SearchResponse> lastAction = searchResponse -> {
+            ClearScrollRequestBuilder clearScrollRequestBuilder =
+                    new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
+                            .addScrollId(searchResponse.getScrollId());
+            clearScrollRequestBuilder.execute().actionGet();
+        };
+        return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
+                condition, lastAction), false)
+                .onClose(infiniteResponses::close)
+                .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
+    }
+
+    @Override
+    public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
+        return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
+    }
+
+    static class TakeWhileSpliterator<T> implements Spliterator<T> {
+
+        private final Spliterator<T> source;
+
+        private final Predicate<T> condition;
+
+        private final Consumer<T> lastElement;
+
+        private final boolean inclusive;
+
+        private final AtomicBoolean checked;
+
+        static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
+                                                Predicate<T> condition,
+                                                Consumer<T> lastElement) {
+            return new TakeWhileSpliterator<>(source, condition, lastElement, true);
+        }
+
+        private TakeWhileSpliterator(Spliterator<T> source,
+                                     Predicate<T> condition,
+                                     Consumer<T> lastElement,
+                                     boolean inclusive) {
+            this.source = source;
+            this.condition = condition;
+            this.lastElement = lastElement;
+            this.inclusive = inclusive;
+            this.checked = new AtomicBoolean(true);
+        }
+
+        @Override
+        public boolean tryAdvance(Consumer<? super T> action) {
+            return checked.get() && source.tryAdvance(e -> {
+                if (condition.test(e)) {
+                    action.accept(e);
+                } else {
+                    if (inclusive && checked.get()) {
+                        action.accept(e);
+                    }
+                    lastElement.accept(e);
+                    checked.set(false);
+                }
+            });
+        }
+
+        @Override
+        public Spliterator<T> trySplit() {
+            return null;
+        }
+
+        @Override
+        public long estimateSize() {
+            return checked.get() ? source.estimateSize() : 0;
+        }
+
+        @Override
+        public int characteristics() {
+            return source.characteristics() &~ Spliterator.SIZED;
+        }
+
+        @Override
+        public Comparator<? super T> getComparator() {
+            return source.getComparator();
+        }
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java

@@ -8,12 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.xbib.elx.api.ExtendedClient;
-import org.xbib.elx.api.ExtendedClientProvider;
+import org.xbib.elx.api.AdminClientProvider;
+import org.xbib.elx.api.BulkClientProvider;
+import org.xbib.elx.api.NativeClient;
+import org.xbib.elx.api.SearchClientProvider;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.ServiceLoader;
 
 @SuppressWarnings("rawtypes")
@@ -23,11 +23,15 @@ public class ClientBuilder {
 
     private final ElasticsearchClient client;
 
+    private final ClassLoader classLoader;
+
     private final Settings.Builder settingsBuilder;
 
-    private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap;
+    private Class<? extends AdminClientProvider> adminClientProvider;
+
+    private Class<? extends BulkClientProvider> bulkClientProvider;
 
-    private Class<? extends ExtendedClientProvider> provider;
+    private Class<? extends SearchClientProvider> searchClientProvider;
 
     public ClientBuilder() {
         this(null);
@@ -39,14 +43,9 @@ public class ClientBuilder {
 
     public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
         this.client = client;
+        this.classLoader = classLoader;
         this.settingsBuilder = Settings.builder();
         settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
-        this.providerMap = new HashMap<>();
-        ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
-                classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
-        for (ExtendedClientProvider provider : serviceLoader) {
-            providerMap.put(provider.getClass(), provider);
-        }
     }
 
     public static ClientBuilder builder() {
@@ -57,8 +56,18 @@ public class ClientBuilder {
         return new ClientBuilder(client);
     }
 
-    public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) {
-        this.provider = provider;
+    public ClientBuilder setAdminClientProvider(Class<? extends AdminClientProvider> adminClientProvider) {
+        this.adminClientProvider = adminClientProvider;
+        return this;
+    }
+
+    public ClientBuilder setBulkClientProvider(Class<? extends BulkClientProvider> bulkClientProvider) {
+        this.bulkClientProvider = bulkClientProvider;
+        return this;
+    }
+
+    public ClientBuilder setSearchClientProvider(Class<? extends SearchClientProvider> searchClientProvider) {
+        this.searchClientProvider = searchClientProvider;
         return this;
     }
 
@@ -98,14 +107,39 @@ public class ClientBuilder {
     }
 
     @SuppressWarnings("unchecked")
-    public <C extends ExtendedClient> C build() throws IOException {
-        if (provider == null) {
-            throw new IllegalArgumentException("no provider");
-        }
+    public <C extends NativeClient> C build() throws IOException {
         Settings settings = settingsBuilder.build();
         logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
-        return (C) providerMap.get(provider).getExtendedClient()
-                .setClient(client)
-                .init(settings);
+        if (adminClientProvider != null) {
+            for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) {
+                if (provider.getClass().isAssignableFrom(adminClientProvider)) {
+                    C c = (C) provider.getClient();
+                    c.setClient(client);
+                    c.init(settings);
+                    return c;
+                }
+            }
+        }
+        if (bulkClientProvider != null) {
+            for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) {
+                if (provider.getClass().isAssignableFrom(bulkClientProvider)) {
+                    C c = (C) provider.getClient();
+                    c.setClient(client);
+                    c.init(settings);
+                    return c;
+                }
+            }
+        }
+        if (searchClientProvider != null) {
+            for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) {
+                if (provider.getClass().isAssignableFrom(searchClientProvider)) {
+                    C c = (C) provider.getClient();
+                    c.setClient(client);
+                    c.init(settings);
+                    return c;
+                }
+            }
+        }
+        throw new IllegalArgumentException("no provider");
     }
 }

elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java

@@ -2,17 +2,19 @@ package org.xbib.elx.common;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.bulk.BulkItemResponse;
+import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.xbib.elx.api.BulkClient;
 import org.xbib.elx.api.BulkController;
-import org.xbib.elx.api.BulkListener;
 import org.xbib.elx.api.BulkMetric;
 import org.xbib.elx.api.BulkProcessor;
-import org.xbib.elx.api.ExtendedClient;
 import org.xbib.elx.api.IndexDefinition;
 
 import java.io.IOException;
@@ -27,12 +29,10 @@ public class DefaultBulkController implements BulkController {
 
     private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
 
-    private final ExtendedClient client;
+    private final BulkClient client;
 
     private final BulkMetric bulkMetric;
 
-    private BulkProcessor bulkProcessor;
-
     private final List<String> indexNames;
 
     private final Map<String, Long> startBulkRefreshIntervals;
@@ -43,11 +43,17 @@ public class DefaultBulkController implements BulkController {
 
     private final TimeUnit maxWaitTimeUnit;
 
-    private final AtomicBoolean active;
+    private BulkProcessor bulkProcessor;
+
+    private BulkListener bulkListener;
+
+    private AtomicBoolean active;
+
+    private boolean enableBulkLogging;
 
-    public DefaultBulkController(ExtendedClient client) {
+    public DefaultBulkController(ExtendedClient client, BulkMetric bulkMetric) {
         this.client = client;
-        this.bulkMetric = new DefaultBulkMetric();
+        this.bulkMetric = bulkMetric;
         this.indexNames = new ArrayList<>();
         this.active = new AtomicBoolean(false);
         this.startBulkRefreshIntervals = new HashMap<>();
@@ -56,14 +62,9 @@ public class DefaultBulkController implements BulkController {
         this.maxWaitTimeUnit = TimeUnit.SECONDS;
     }
 
-    @Override
-    public BulkMetric getBulkMetric() {
-        return bulkMetric;
-    }
-
     @Override
     public Throwable getLastBulkError() {
-        return bulkProcessor.getBulkListener().getLastBulkError();
+        return bulkListener.getLastBulkError();
     }
 
     @Override
@@ -77,27 +78,22 @@ public class DefaultBulkController implements BulkController {
         ByteSizeValue maxVolumePerRequest = settings.getAsBytesSize(Parameters.MAX_VOLUME_PER_REQUEST.name(),
                 ByteSizeValue.parseBytesSizeValue(Parameters.DEFAULT_MAX_VOLUME_PER_REQUEST.getString(),
                         "maxVolumePerRequest"));
-        boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(),
+        this.enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(),
                 Parameters.ENABLE_BULK_LOGGING.getValue());
-        BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging);
+        this.bulkListener = new BulkListener();
         this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener)
                 .setBulkActions(maxActionsPerRequest)
                 .setConcurrentRequests(maxConcurrentRequests)
                 .setFlushInterval(flushIngestInterval)
                 .setBulkSize(maxVolumePerRequest)
                 .build();
+        this.active.set(true);
         if (logger.isInfoEnabled()) {
             logger.info("bulk processor up with maxActionsPerRequest = {} maxConcurrentRequests = {} " +
-                            "flushIngestInterval = {} maxVolumePerRequest = {}, bulk logging = {}",
+                            "flushIngestInterval = {} maxVolumePerRequest = {} bulk logging = {} logger debug = {} from settings = {}",
                     maxActionsPerRequest, maxConcurrentRequests, flushIngestInterval, maxVolumePerRequest,
-                    enableBulkLogging);
+                    enableBulkLogging, logger.isDebugEnabled(), settings.toDelimitedString(','));
         }
-        this.active.set(true);
-    }
-
-    @Override
-    public void inactivate() {
-        this.active.set(false);
     }
 
     @Override
@@ -122,53 +118,65 @@ public class DefaultBulkController implements BulkController {
     }
 
     @Override
-    public void bulkIndex(IndexRequest indexRequest) {
+    public void index(IndexRequest indexRequest) {
         ensureActiveAndBulk();
+        if (!active.get()) {
+            throw new IllegalStateException("inactive");
+        }
         try {
-            bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
+            if (bulkMetric != null) {
+                bulkMetric.getCurrentIngest().inc(indexRequest.index(), indexRequest.type(), indexRequest.id());
+            }
             bulkProcessor.add(indexRequest);
         } catch (Exception e) {
+            bulkListener.lastBulkError = e;
+            active.set(false);
             if (logger.isErrorEnabled()) {
                 logger.error("bulk add of index failed: " + e.getMessage(), e);
             }
-            inactivate();
         }
     }
 
     @Override
-    public void bulkDelete(DeleteRequest deleteRequest) {
+    public void delete(DeleteRequest deleteRequest) {
         if (!active.get()) {
             throw new IllegalStateException("inactive");
         }
         try {
-            bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
+            if (bulkMetric != null) {
+                bulkMetric.getCurrentIngest().inc(deleteRequest.index(), deleteRequest.type(), deleteRequest.id());
+            }
             bulkProcessor.add(deleteRequest);
         } catch (Exception e) {
+            bulkListener.lastBulkError = e;
+            active.set(false);
             if (logger.isErrorEnabled()) {
                 logger.error("bulk add of delete failed: " + e.getMessage(), e);
             }
-            inactivate();
         }
     }
 
     @Override
-    public void bulkUpdate(UpdateRequest updateRequest) {
+    public void update(UpdateRequest updateRequest) {
         if (!active.get()) {
             throw new IllegalStateException("inactive");
         }
         try {
-            bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id());
+            if (bulkMetric != null) {
+                bulkMetric.getCurrentIngest().inc(updateRequest.index(), updateRequest.type(), updateRequest.id());
+            }
             bulkProcessor.add(updateRequest);
         } catch (Exception e) {
+            bulkListener.lastBulkError = e;
+            active.set(false);
             if (logger.isErrorEnabled()) {
                 logger.error("bulk add of update failed: " + e.getMessage(), e);
             }
-            inactivate();
         }
     }
 
     @Override
-    public boolean waitForBulkResponses(long timeout, TimeUnit timeUnit) {
+    public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
         try {
             return bulkProcessor.awaitFlush(timeout, timeUnit);
         } catch (InterruptedException e) {
@@ -187,7 +195,7 @@ public class DefaultBulkController implements BulkController {
     @Override
     public void stopBulkMode(String index, long timeout, TimeUnit timeUnit) throws IOException {
         flush();
-        if (waitForBulkResponses(timeout, timeUnit)) {
+        if (waitForResponses(timeout, timeUnit)) {
             if (indexNames.contains(index)) {
                 Long secs = stopBulkRefreshIntervals.get(index);
                 if (secs != null && secs != 0L) {
@@ -209,7 +217,6 @@ public class DefaultBulkController implements BulkController {
     @Override
     public void close() throws IOException {
         flush();
-        bulkMetric.close();
         if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) {
             for (String index : indexNames) {
                 Long secs = stopBulkRefreshIntervals.get(index);
@@ -231,5 +238,92 @@ public class DefaultBulkController implements BulkController {
         if (bulkProcessor == null) {
             throw new UnsupportedOperationException("bulk processor not present");
         }
+        if (bulkListener == null) {
+            throw new UnsupportedOperationException("bulk listener not present");
+        }
+    }
+
+    private class BulkListener implements DefaultBulkProcessor.Listener {
+
+        private final Logger logger = LogManager.getLogger(BulkListener.class.getName());
+
+        private Throwable lastBulkError = null;
+
+        @Override
+        public void beforeBulk(long executionId, BulkRequest request) {
+            long l = 0;
+            if (bulkMetric != null) {
+                l = bulkMetric.getCurrentIngest().getCount();
+                bulkMetric.getCurrentIngest().inc();
+                int n = request.numberOfActions();
+                bulkMetric.getSubmitted().inc(n);
+                bulkMetric.getCurrentIngestNumDocs().inc(n);
+                bulkMetric.getTotalIngestSizeInBytes().inc(request.estimatedSizeInBytes());
+            }
+            if (enableBulkLogging && logger.isDebugEnabled()) {
+                logger.debug("before bulk [{}] [actions={}] [bytes={}] [concurrent requests={}]",
+                        executionId,
+                        request.numberOfActions(),
+                        request.estimatedSizeInBytes(),
+                        l);
+            }
+        }
+
+        @Override
+        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
+            long l = 0;
+            if (bulkMetric != null) {
+                l = bulkMetric.getCurrentIngest().getCount();
+                bulkMetric.getCurrentIngest().dec();
+                bulkMetric.getSucceeded().inc(response.getItems().length);
+            }
+            int n = 0;
+            for (BulkItemResponse itemResponse : response.getItems()) {
+                if (bulkMetric != null) {
+                    bulkMetric.getCurrentIngest().dec(itemResponse.getIndex(), itemResponse.getType(), itemResponse.getId());
+                }
+                if (itemResponse.isFailed()) {
+                    n++;
+                    if (bulkMetric != null) {
+                        bulkMetric.getSucceeded().dec(1);
+                        bulkMetric.getFailed().inc(1);
+                    }
+                }
+            }
+            if (enableBulkLogging && logger.isDebugEnabled() && bulkMetric != null) {
+                logger.debug("after bulk [{}] [succeeded={}] [failed={}] [{}ms] {} concurrent requests",
+                        executionId,
+                        bulkMetric.getSucceeded().getCount(),
+                        bulkMetric.getFailed().getCount(),
+                        response.getTook().millis(),
+                        l);
+            }
+            if (n > 0) {
+                if (enableBulkLogging && logger.isErrorEnabled()) {
+                    logger.error("bulk [{}] failed with {} failed items, failure message = {}",
+                            executionId, n, response.buildFailureMessage());
+                }
+            } else {
+                if (bulkMetric != null) {
+                    bulkMetric.getCurrentIngestNumDocs().dec(response.getItems().length);
+                }
+            }
+        }
+
+        @Override
+        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
+            if (bulkMetric != null) {
+                bulkMetric.getCurrentIngest().dec();
+            }
+            lastBulkError = failure;
+            active.set(false);
+            if (enableBulkLogging && logger.isErrorEnabled()) {
+                logger.error("after bulk [" + executionId + "] error", failure);
+            }
+        }
+
+        Throwable getLastBulkError() {
+            return lastBulkError;
+        }
     }
 }

elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java

@@ -3,8 +3,6 @@ package org.xbib.elx.common;
 import org.xbib.elx.api.IndexDefinition;
 import org.xbib.elx.api.IndexRetention;
 
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.concurrent.TimeUnit;
 
 public class DefaultIndexDefinition implements IndexDefinition {
@@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
 
     private String dateTimePattern;
 
-    private URL settingsUrl;
+    private String settings;
 
-    private URL mappingsUrl;
+    private String mappings;
 
     private boolean enabled;
 
@@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
     }
 
     @Override
-    public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException {
-        this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null;
+    public IndexDefinition setSettings(String settings) {
+        this.settings = settings;
         return this;
     }
 
     @Override
-    public IndexDefinition setSettingsUrl(URL settingsUrl) {
-        this.settingsUrl = settingsUrl;
-        return this;
-    }
-
-    @Override
-    public URL getSettingsUrl() {
-        return settingsUrl;
-    }
-
-    @Override
-    public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException {
-        this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null;
-        return this;
+    public String getSettings() {
+        return settings;
     }
 
     @Override
-    public IndexDefinition setMappingsUrl(URL mappingsUrl) {
-        this.mappingsUrl = mappingsUrl;
+    public IndexDefinition setMappings(String mappings) {
+        this.mappings = mappings;
         return this;
     }
 
     @Override
-    public URL getMappingsUrl() {
-        return mappingsUrl;
+    public String getMappings() {
+        return mappings;
     }
 
     @Override

elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java

@@ -0,0 +1,59 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A mocked client, it does not perform any actions on a cluster. Useful for testing.
+ */
+public class MockAdminClient extends AbstractAdminClient {
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return null;
+    }
+
+    @Override
+    public void init(Settings settings) {
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) {
+        return null;
+    }
+
+    @Override
+    protected void closeClient() {
+    }
+
+    @Override
+    public MockAdminClient deleteIndex(String index) {
+        return this;
+    }
+
+    @Override
+    public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
+        return true;
+    }
+
+    @Override
+    public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
+    }
+
+    @Override
+    public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
+        return true;
+    }
+
+    @Override
+    public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
+        return this;
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java

@@ -0,0 +1,10 @@
+package org.xbib.elx.common;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
+    @Override
+    public MockAdminClient getClient() {
+        return new MockAdminClient();
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java

@@ -0,0 +1,99 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A mocked client, it does not perform any actions on a cluster. Useful for testing.
+ */
+public class MockBulkClient extends AbstractBulkClient {
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return null;
+    }
+
+    @Override
+    public void init(Settings settings) {
+    }
+
+    @Override
+    public String getClusterName() {
+        return null;
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) {
+        return null;
+    }
+
+    @Override
+    protected void closeClient() {
+    }
+
+    @Override
+    public MockBulkClient index(String index, String id, boolean create, String source) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient delete(String index, String id) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient update(String index, String id, String source) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient index(IndexRequest indexRequest) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient delete(DeleteRequest deleteRequest) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient update(UpdateRequest updateRequest) {
+        return this;
+    }
+
+    @Override
+    public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
+    }
+
+    @Override
+    public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
+    }
+
+    @Override
+    public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
+        return true;
+    }
+
+    @Override
+    public void refreshIndex(String index) {
+    }
+
+    @Override
+    public void flushIndex(String index) {
+    }
+
+
+    @Override
+    public void flush() {
+        // nothing to do
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.common;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
+
+    @Override
+    public MockBulkClient getClient() {
+        return new MockBulkClient();
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java

@@ -1,129 +0,0 @@
-package org.xbib.elx.common;
-
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.settings.Settings;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * A mocked client, it does not perform any actions on a cluster. Useful for testing.
- */
-public class MockExtendedClient extends AbstractExtendedClient {
-
-    @Override
-    public ElasticsearchClient getClient() {
-        return null;
-    }
-
-    @Override
-    public MockExtendedClient init(Settings settings) {
-        return this;
-    }
-
-    @Override
-    protected ElasticsearchClient createClient(Settings settings) {
-        return null;
-    }
-
-    @Override
-    protected void closeClient() {
-    }
-
-    @Override
-    public MockExtendedClient index(String index, String id, boolean create, String source) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient delete(String index, String id) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient update(String index, String id, String source) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient index(IndexRequest indexRequest) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient delete(DeleteRequest deleteRequest) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient update(UpdateRequest updateRequest) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient newIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient deleteIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient refreshIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient flushIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
-        return this;
-    }
-
-    @Override
-    public void flush() {
-        // nothing to do
-    }
-
-    @Override
-    public void close() {
-        // nothing to do
-    }
-}

elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java

@@ -1,10 +0,0 @@
-package org.xbib.elx.common;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
-    @Override
-    public MockExtendedClient getExtendedClient() {
-        return new MockExtendedClient();
-    }
-}

elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java

@@ -0,0 +1,38 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+
+/**
+ * A mocked client, it does not perform any actions on a cluster. Useful for testing.
+ */
+public class MockSearchClient extends AbstractSearchClient {
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return null;
+    }
+
+    @Override
+    public void init(Settings settings) {
+    }
+
+    @Override
+    public String getClusterName() {
+        return null;
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) {
+        return null;
+    }
+
+    @Override
+    protected void closeClient() {
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.common;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
+
+    @Override
+    public MockSearchClient getClient() {
+        return new MockSearchClient();
+    }
+}

elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.common.MockAdminClientProvider

elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.common.MockExtendedClientProvider

elx-common/src/test/java/org/xbib/elx/common/test/MockAdminClientProviderTest.java

@@ -0,0 +1,21 @@
+package org.xbib.elx.common.test;
+
+import org.junit.jupiter.api.Test;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.MockAdminClient;
+import org.xbib.elx.common.MockAdminClientProvider;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+class MockAdminClientProviderTest {
+
+    @Test
+    void testMockAdminProvider() throws IOException {
+        MockAdminClient client = ClientBuilder.builder()
+                .setAdminClientProvider(MockAdminClientProvider.class)
+                .build();
+        assertNotNull(client);
+    }
+}

elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java

@@ -1,19 +0,0 @@
-package org.xbib.elx.common.test;
-
-import org.junit.jupiter.api.Test;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.common.MockExtendedClient;
-import org.xbib.elx.common.MockExtendedClientProvider;
-
-import java.io.IOException;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-class MockExtendedClientProviderTest {
-
-    @Test
-    void testMockExtendedProvider() throws IOException {
-        MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
-        assertNotNull(client);
-    }
-}

elx-http/src/main/java/org/elasticsearch/action/admin/indices/get/HttpGetIndexAction.java

@@ -0,0 +1,155 @@
+package org.elasticsearch.action.admin.indices.get;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+import org.apache.lucene.util.CollectionUtil;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elx.http.HttpAction;
+import org.xbib.netty.http.client.api.Request;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+public class HttpGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
+
+    @Override
+    public GetIndexAction getActionInstance() {
+        return GetIndexAction.INSTANCE;
+    }
+
+    @Override
+    protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
+        List<String> list =  getIndexRequest.indices().length == 0 ?
+                List.of("*") : Arrays.asList(getIndexRequest.indices());
+        String command = "/" + String.join(",", list);
+        logger.info("command = " + command);
+        return newGetRequest(url, command);
+    }
+
+    @Override
+    protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser() {
+        return this::fromXContent;
+    }
+
+    @Override
+    protected GetIndexResponse emptyResponse() {
+        return new GetIndexResponse();
+    }
+
+    private GetIndexResponse fromXContent(XContentParser parser) throws IOException {
+        ImmutableOpenMap.Builder<String, List<AliasMetaData>> aliases = ImmutableOpenMap.builder();
+        ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> mappings = ImmutableOpenMap.builder();
+        ImmutableOpenMap.Builder<String, Settings> settings = ImmutableOpenMap.builder();
+        List<String> indices = new ArrayList<>();
+        if (parser.currentToken() == null) {
+            parser.nextToken();
+        }
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        parser.nextToken();
+        while (!parser.isClosed()) {
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                String indexName = parser.currentName();
+                indices.add(indexName);
+                IndexEntry indexEntry = parseIndexEntry(parser);
+                CollectionUtil.timSort(indexEntry.indexAliases, Comparator.comparing(AliasMetaData::alias));
+                aliases.put(indexName, Collections.unmodifiableList(indexEntry.indexAliases));
+                mappings.put(indexName, indexEntry.indexMappings);
+                settings.put(indexName, indexEntry.indexSettings);
+            } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                parser.skipChildren();
+            } else {
+                parser.nextToken();
+            }
+        }
+        return new GetIndexResponse(indices.toArray(new String[0]),
+                mappings.build(),
+                aliases.build(),
+                settings.build());
+    }
+
+    private static IndexEntry parseIndexEntry(XContentParser parser) throws IOException {
+        List<AliasMetaData> indexAliases = null;
+        ImmutableOpenMap<String, MappingMetaData> indexMappings = null;
+        Settings indexSettings = null;
+        Settings indexDefaultSettings = null;
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+            parser.nextToken();
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                switch (parser.currentName()) {
+                    case "aliases":
+                        indexAliases = parseAliases(parser);
+                        break;
+                    case "mappings":
+                        indexMappings = parseMappings(parser);
+                        break;
+                    case "settings":
+                        indexSettings = Settings.fromXContent(parser);
+                        break;
+                    case "defaults":
+                        indexDefaultSettings = Settings.fromXContent(parser);
+                        break;
+                    default:
+                        parser.skipChildren();
+                }
+            } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                parser.skipChildren();
+            }
+        }
+        return new IndexEntry(indexAliases, indexMappings, indexSettings, indexDefaultSettings);
+    }
+
+    private static List<AliasMetaData> parseAliases(XContentParser parser) throws IOException {
+        List<AliasMetaData> indexAliases = new ArrayList<>();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+            indexAliases.add(AliasMetaData.Builder.fromXContent(parser));
+        }
+        return indexAliases;
+    }
+
+    private static ImmutableOpenMap<String, MappingMetaData> parseMappings(XContentParser parser) throws IOException {
+        ImmutableOpenMap.Builder<String, MappingMetaData> indexMappings = ImmutableOpenMap.builder();
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation);
+            parser.nextToken();
+            if (parser.currentToken() == XContentParser.Token.START_OBJECT) {
+                String mappingType = parser.currentName();
+                indexMappings.put(mappingType, new MappingMetaData(mappingType, parser.map()));
+            } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) {
+                parser.skipChildren();
+            }
+        }
+        return indexMappings.build();
+    }
+
+    private static class IndexEntry {
+        List<AliasMetaData> indexAliases = new ArrayList<>();
+        ImmutableOpenMap<String, MappingMetaData> indexMappings = ImmutableOpenMap.of();
+        Settings indexSettings = Settings.EMPTY;
+        Settings indexDefaultSettings = Settings.EMPTY;
+
+        IndexEntry(List<AliasMetaData> indexAliases, ImmutableOpenMap<String, MappingMetaData> indexMappings,
+                   Settings indexSettings, Settings indexDefaultSettings) {
+            if (indexAliases != null) {
+                this.indexAliases = indexAliases;
+            }
+            if (indexMappings != null) {
+                this.indexMappings = indexMappings;
+            }
+            if (indexSettings != null) {
+                this.indexSettings = indexSettings;
+            }
+            if (indexDefaultSettings != null) {
+                this.indexDefaultSettings = indexDefaultSettings;
+            }
+        }
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java

@@ -1,10 +0,0 @@
-package org.xbib.elx.http;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
-    @Override
-    public ExtendedHttpClient getExtendedClient() {
-        return new ExtendedHttpClient();
-    }
-}

elx-http/src/main/java/org/xbib/elx/http/HttpAction.java

@@ -154,15 +154,35 @@ public abstract class HttpAction<R extends ActionRequest, T extends ActionRespon
         if (xContentType == null) {
             throw new IllegalStateException("unsupported content-type: " + mediaType);
         }
-        String body = httpActionContext.getHttpResponse().content().toString(StandardCharsets.UTF_8);
-        T t;
-        try (XContentParser parser = xContentType.xContent().createParser(body)) {
-            t = entityParser().apply(parser);
+        try (XContentParser parser = xContentType.xContent()
+                .createParser(httpActionContext.getExtendedHttpClient().getRegistry(),
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
+            return entityParser().apply(parser);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            return null;
         }
         return t;
     }
 
-    protected abstract RequestBuilder createHttpRequest(String baseUrl, R request) throws IOException;
+    protected ElasticsearchStatusException parseToError(HttpActionContext<R, T> httpActionContext) {
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+                .createParser(httpActionContext.getExtendedHttpClient().getRegistry(),
+                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
+                        httpActionContext.getHttpResponse().getBody().toString(StandardCharsets.UTF_8))) {
+            return errorParser().apply(parser);
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+            return new ElasticsearchStatusException(e.getMessage(), RestStatus.INTERNAL_SERVER_ERROR, e);
+        }
+    }
+
+    protected CheckedFunction<XContentParser, ElasticsearchStatusException, IOException> errorParser() {
+        return BytesRestResponse::errorFromXContent;
+    }
+
+    protected abstract Request.Builder createHttpRequest(String baseUrl, R request) throws IOException;
 
     protected abstract CheckedFunction<XContentParser, T, IOException> entityParser();
 

elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java

@@ -13,7 +13,7 @@ import org.xbib.netty.http.client.transport.Transport;
  */
 public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
 
-    private final ExtendedHttpClient extendedHttpClient;
+    private final HttpClientHelper helper;
 
     private final R request;
 
@@ -23,14 +23,14 @@ public class HttpActionContext<R extends ActionRequest, T extends ActionResponse
 
     private FullHttpResponse httpResponse;
 
-    HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) {
-        this.extendedHttpClient = extendedHttpClient;
+    HttpActionContext(HttpClientHelper helper, R request, String url) {
+        this.helper = helper;
         this.request = request;
         this.url = url;
     }
 
-    public ExtendedHttpClient getExtendedHttpClient() {
-        return extendedHttpClient;
+    public HttpClientHelper getHelper() {
+        return helper;
     }
 
     public R getRequest() {

elx-http/src/main/java/org/xbib/elx/http/HttpAdminClient.java

@@ -0,0 +1,61 @@
+package org.xbib.elx.http;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elx.common.AbstractAdminClient;
+import java.io.IOException;
+
+/**
+ * Elasticsearch HTTP admin client.
+ */
+public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
+
+    private final HttpClientHelper helper;
+
+    public HttpAdminClient() {
+        this.helper = new HttpClientHelper();
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init(settings);
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    protected void closeClient() throws IOException {
+        helper.closeClient();
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
+        return helper.execute(action, request);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+        helper.execute(action, request, listener);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
+        return helper.prepareExecute(action);
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return helper.threadPool();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java

@@ -0,0 +1,10 @@
+package org.xbib.elx.http;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
+    @Override
+    public HttpAdminClient getClient() {
+        return new HttpAdminClient();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java

@@ -0,0 +1,61 @@
+package org.xbib.elx.http;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elx.common.AbstractBulkClient;
+import java.io.IOException;
+
+/**
+ * Elasticsearch HTTP bulk client.
+ */
+public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
+
+    private final HttpClientHelper helper;
+
+    public HttpBulkClient() {
+        this.helper = new HttpClientHelper();
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init(settings);
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    protected void closeClient() throws IOException {
+        helper.closeClient();
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
+        return helper.execute(action, request);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+        helper.execute(action, request, listener);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
+        return helper.prepareExecute(action);
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return helper.threadPool();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.http;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
+
+    @Override
+    public HttpBulkClient getClient() {
+        return new HttpBulkClient();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java

@@ -1,5 +1,6 @@
 package org.xbib.elx.http;
 
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.Action;
@@ -12,52 +13,79 @@ import org.elasticsearch.action.GenericAction;
 import org.elasticsearch.action.support.PlainActionFuture;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.xbib.elx.common.AbstractExtendedClient;
+import org.xbib.elx.common.AbstractAdminClient;
+import org.xbib.net.URL;
 import org.xbib.netty.http.client.Client;
-
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.ServiceLoader;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Elasticsearch HTTP client.
  */
-public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient {
+public class HttpClientHelper extends AbstractAdminClient implements ElasticsearchClient {
 
-    private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class);
+    private static final Logger logger = LogManager.getLogger(HttpClientHelper.class);
 
     private Client nettyHttpClient;
 
     private final ClassLoader classLoader;
 
+    private final NamedXContentRegistry registry;
+
     @SuppressWarnings("rawtypes")
     private final Map<GenericAction, HttpAction> actionMap;
 
     private String url;
 
-    public ExtendedHttpClient() {
-        this.classLoader = ExtendedHttpClient.class.getClassLoader();
+    public ExtendedHttpClient(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
+        this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
+                namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
+        this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
         this.actionMap = new HashMap<>();
     }
 
     @Override
     @SuppressWarnings({"unchecked", "rawtypes"})
-    public ExtendedHttpClient init(Settings settings) throws IOException {
+    public void init(Settings settings) throws IOException {
         super.init(settings);
-        if (settings == null) {
-            return null;
+        if (settings.hasValue("url")) {
+            this.url = settings.get("url");
+        } else if (settings.hasValue("host")) {
+            this.url = URL.http()
+                    .host(settings.get("host")).port(settings.getAsInt("port", 9200))
+                    .build()
+                    .toExternalForm();
         }
-        this.url = settings.get("url");
         ServiceLoader<HttpAction> httpActionServiceLoader = ServiceLoader.load(HttpAction.class, classLoader);
         for (HttpAction<? extends ActionRequest, ? extends ActionResponse> httpAction : httpActionServiceLoader) {
             httpAction.setSettings(settings);
             actionMap.put(httpAction.getActionInstance(), httpAction);
         }
-        this.nettyHttpClient = Client.builder().enableDebug().build();
-        logger.info("extended HTTP client initialized with {} actions", actionMap.size());
-        return this;
+        Client.Builder clientBuilder = Client.builder();
+        if (settings.hasValue("debug")) {
+            clientBuilder.enableDebug();
+        }
+        this.nettyHttpClient = clientBuilder.build();
+        logger.log(Level.DEBUG, "extended HTTP client initialized, settings = {}, url = {}, {} actions",
+                settings, url, actionMap.size());
+    }
+
+    private static List<NamedXContentRegistry.Entry> getNamedXContents() {
+        return new ArrayList<>();
+    }
+
+    public NamedXContentRegistry getRegistry() {
+        return registry;
     }
 
     public Client internalClient() {
@@ -104,7 +132,8 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
 
     @Override
     public ThreadPool threadPool() {
-        throw new UnsupportedOperationException();
+        logger.log(Level.DEBUG, "returning null for threadPool() request");
+        return null;
     }
 
     @SuppressWarnings({"unchecked", "rawtypes"})
@@ -116,9 +145,7 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
         }
         try {
             HttpActionContext httpActionContext = new HttpActionContext(this, request, url);
-            if (logger.isDebugEnabled()) {
-                logger.debug("submitting request {} to URL {}", request, url);
-            }
+            logger.log(Level.DEBUG, "url = " + url);
             httpAction.execute(httpActionContext, listener);
         } catch (Exception e) {
             logger.error(e.getMessage(), e);

elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java

@@ -0,0 +1,61 @@
+package org.xbib.elx.http;
+
+import org.elasticsearch.action.Action;
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elx.common.AbstractSearchClient;
+import java.io.IOException;
+
+/**
+ * Elasticsearch HTTP search client.
+ */
+public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
+
+    private final HttpClientHelper helper;
+
+    public HttpSearchClient() {
+        this.helper = new HttpClientHelper();
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init(settings);
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    protected void closeClient() throws IOException {
+        helper.closeClient();
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> ActionFuture<Response> execute(Action<Request, Response, RequestBuilder> action, Request request) {
+        return helper.execute(action, request);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void execute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
+        helper.execute(action, request, listener);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> RequestBuilder prepareExecute(Action<Request, Response, RequestBuilder> action) {
+        return helper.prepareExecute(action);
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return helper.threadPool();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.http;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
+
+    @Override
+    public HttpSearchClient getClient() {
+        return new HttpSearchClient();
+    }
+}

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.http.HttpAdminClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.http.HttpBulkClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.http.ExtendedHttpClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.http.HttpSearchClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction

@@ -1,3 +1,18 @@
+org.elasticsearch.action.admin.indices.mapping.get.HttpGetMappingsAction
+org.xbib.elx.http.action.admin.cluster.health.HttpClusterHealthAction
+org.xbib.elx.http.action.admin.cluster.node.info.HttpNodesInfoAction
+org.xbib.elx.http.action.admin.cluster.settings.HttpClusterUpdateSettingsAction
+org.xbib.elx.http.action.admin.cluster.state.HttpClusterStateAction
+org.xbib.elx.http.action.admin.indices.alias.HttpIndicesAliasesAction
+org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction
+org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
+org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
+org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
+org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction
+org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction
+org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction
+org.xbib.elx.http.action.bulk.HttpBulkAction
+org.xbib.elx.http.action.index.HttpIndexAction
 org.xbib.elx.http.action.search.HttpSearchAction
 org.xbib.elx.http.action.get.HttpGetAction
 org.xbib.elx.http.action.get.HttpMultiGetAction

elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java

@@ -1,4 +1,4 @@
-package org.xbib.elx.node.test;
+package org.xbib.elx.http.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -8,21 +8,21 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryBuilders;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.common.Parameters;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
 
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -32,11 +32,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 @ExtendWith(TestExtension.class)
 class ClientTest {
 
-    private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
+    private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
 
-    private static final Long ACTIONS = 1000L;
+    private static final Long ACTIONS = 100L;
 
-    private static final Long MAX_ACTIONS_PER_REQUEST = 100L;
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
 
     private final TestExtension.Helper helper;
 
@@ -44,20 +44,45 @@ class ClientTest {
         this.helper = helper;
     }
 
+    @Test
+    void testSingleDoc() throws Exception {
+        final ExtendedHttpClient client = ClientBuilder.builder()
+                .provider(ExtendedHttpClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
+                .build();
+        try {
+            client.newIndex("test");
+            client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            client.flush();
+            client.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
+            if (client.getBulkController().getLastBulkError() != null) {
+                logger.error("error", client.getBulkController().getLastBulkError());
+            }
+            assertNull(client.getBulkController().getLastBulkError());
+            client.close();
+        }
+    }
+
     @Test
     void testNewIndex() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final ExtendedHttpClient client = ClientBuilder.builder()
+                .provider(ExtendedHttpClientProvider.class)
+                .put(helper.getHttpSettings())
                 .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
                 .build();
-        client.newIndex("test1");
+        client.newIndex("test");
         client.close();
     }
 
     @Test
     void testMapping() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final ExtendedHttpClient client = ClientBuilder.builder()
+                .provider(ExtendedHttpClientProvider.class)
+                .put(helper.getHttpSettings())
                 .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
                 .build();
         XContentBuilder builder = JsonXContent.contentBuilder()
@@ -70,68 +95,44 @@ class ClientTest {
                 .endObject()
                 .endObject()
                 .endObject();
-        client.newIndex("test2", Settings.EMPTY, builder.string());
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2");
+        client.newIndex("test", Settings.EMPTY, Strings.toString(builder));
+        GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test");
         GetMappingsResponse getMappingsResponse =
                 client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
         logger.info("mappings={}", getMappingsResponse.getMappings());
-        assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc"));
+        assertTrue(getMappingsResponse.getMappings().get("test").containsKey("doc"));
         client.close();
     }
 
-    @Test
-    void testSingleDoc() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
-                .build();
-        try {
-            client.newIndex("test3");
-            client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.close();
-        }
-    }
-
     @Test
     void testRandomDocs() throws Exception {
         long numactions = ACTIONS;
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final ExtendedHttpClient client = ClientBuilder.builder()
+                .provider(ExtendedHttpClientProvider.class)
+                .put(helper.getHttpSettings())
                 .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
                 .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
                 .build();
         try {
-            client.newIndex("test4");
+            client.newIndex("test");
             for (int i = 0; i < ACTIONS; i++) {
-                client.index("test4", null, false,
-                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
             client.flush();
-            client.waitForResponses(60L, TimeUnit.SECONDS);
+            client.waitForResponses(30L, TimeUnit.SECONDS);
         } catch (NoNodeAvailableException e) {
             logger.warn("skipping, no node available");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
         } finally {
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
+            assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
             if (client.getBulkController().getLastBulkError() != null) {
                 logger.error("error", client.getBulkController().getLastBulkError());
             }
             assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test4");
+            client.refreshIndex("test");
             SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
-                    .setIndices("test4")
-                    .setQuery(QueryBuilders.matchAllQuery())
-                    .setSize(0);
+                    .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
             assertEquals(numactions,
                     searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
             client.close();
@@ -141,55 +142,60 @@ class ClientTest {
     @Test
     void testThreadedRandomDocs() throws Exception {
         int maxthreads = Runtime.getRuntime().availableProcessors();
-        long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
-        final long actions = ACTIONS;
-        logger.info("NodeClient max={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
+        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
+        final Long actions = ACTIONS;
+        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
+        final ExtendedHttpClient client = ClientBuilder.builder()
+                .provider(ExtendedHttpClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
                 .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
                 .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
                 .build();
         try {
-            client.newIndex("test5")
-                    .startBulk("test5", -1, 1000);
-            ThreadPoolExecutor pool = EsExecutors.newFixed("nodeclient-test", maxthreads, 30,
-                    EsExecutors.daemonThreadFactory("nodeclient-test"));
+            Settings settings = Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .build();
+            client.newIndex("test", settings, (String)null)
+                    .startBulk("test", 0, 1000);
+            logger.info("index created");
+            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
             final CountDownLatch latch = new CountDownLatch(maxthreads);
             for (int i = 0; i < maxthreads; i++) {
-                pool.execute(() -> {
+                executorService.execute(() -> {
                     for (int i1 = 0; i1 < actions; i1++) {
-                        client.index("test5", null, false,
-                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                        client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
                     }
                     latch.countDown();
                 });
             }
             logger.info("waiting for latch...");
-            if (latch.await(60, TimeUnit.SECONDS)) {
+            if (latch.await(60L, TimeUnit.SECONDS)) {
                 logger.info("flush...");
                 client.flush();
                 client.waitForResponses(60L, TimeUnit.SECONDS);
-                logger.info("pool shutdown...");
-                pool.shutdown();
+                logger.info("got all responses, executor service shutdown...");
+                executorService.shutdown();
+                executorService.awaitTermination(60L, TimeUnit.SECONDS);
                 logger.info("pool is shut down");
             } else {
                 logger.warn("latch timeout");
             }
+            client.stopBulk("test", 30L, TimeUnit.SECONDS);
+            assertEquals(maxthreads * actions, client.getBulkMetric().getSucceeded().getCount());
         } catch (NoNodeAvailableException e) {
             logger.warn("skipping, no node available");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
         } finally {
-            client.stopBulk("test5", 60L, TimeUnit.SECONDS);
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
             if (client.getBulkController().getLastBulkError() != null) {
                 logger.error("error", client.getBulkController().getLastBulkError());
             }
             assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test5");
+            client.refreshIndex("test");
             SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
-                    .setIndices("test5")
-                    .setQuery(QueryBuilders.matchAllQuery())
-                    .setSize(0);
+                    .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
             assertEquals(maxthreads * actions,
                     searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
             client.close();

elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java

@@ -1,122 +0,0 @@
-package org.xbib.elx.http.test;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
-import org.elasticsearch.action.get.GetAction;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetAction;
-import org.elasticsearch.action.get.MultiGetRequest;
-import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.index.IndexAction;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.index.IndexResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.http.ExtendedHttpClient;
-import org.xbib.elx.http.ExtendedHttpClientProvider;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class ClientTest extends TestBase {
-
-    private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
-
-    @Ignore
-    @Test
-    public void testGet() throws Exception {
-        try (ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put("url", "http://" + host + ":" + httpPort)
-                .build()) {
-            IndexRequest indexRequest = new IndexRequest();
-            indexRequest.index("test");
-            indexRequest.type("test");
-            indexRequest.id("1");
-            indexRequest.source("test", "Hello Jörg");
-            IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
-            client("1").execute(RefreshAction.INSTANCE, new RefreshRequest());
-
-            GetRequest getRequest = new GetRequest();
-            getRequest.index("test");
-            getRequest.type("test");
-            getRequest.id("1");
-
-            GetResponse getResponse = client.execute(GetAction.INSTANCE, getRequest).actionGet();
-
-            assertTrue(getResponse.isExists());
-            assertEquals("{\"test\":\"Hello Jörg\"}", getResponse.getSourceAsString());
-
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        }
-    }
-
-    @Ignore
-    @Test
-    public void testMultiGet() throws Exception {
-        try (ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put("url", "http://" + host + ":" + httpPort)
-                .build()) {
-            IndexRequest indexRequest = new IndexRequest();
-            indexRequest.index("test");
-            indexRequest.type("test");
-            indexRequest.id("1");
-            indexRequest.source("test", "Hello Jörg");
-            IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
-            client("1").execute(RefreshAction.INSTANCE, new RefreshRequest());
-
-            MultiGetRequest multiGetRequest = new MultiGetRequest();
-            multiGetRequest.add("test", "test", "1");
-
-            MultiGetResponse multiGetResponse = client.execute(MultiGetAction.INSTANCE, multiGetRequest).actionGet();
-
-            assertEquals(1, multiGetResponse.getResponses().length);
-            assertEquals("{\"test\":\"Hello Jörg\"}", multiGetResponse.getResponses()[0].getResponse().getSourceAsString());
-
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        }
-    }
-
-    @Test
-    public void testSearchDoc() throws Exception {
-        try (ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put("url", "http://" + host + ":" + httpPort)
-                .build()) {
-            IndexRequest indexRequest = new IndexRequest();
-            indexRequest.index("test");
-            indexRequest.type("test");
-            indexRequest.id("1");
-            indexRequest.source("test", "Hello Jörg");
-            IndexResponse indexResponse = client("1").execute(IndexAction.INSTANCE, indexRequest).actionGet();
-            client("1").execute(RefreshAction.INSTANCE, new RefreshRequest());
-
-            SearchSourceBuilder builder = new SearchSourceBuilder();
-            builder.query(QueryBuilders.matchAllQuery());
-            SearchRequest searchRequest = new SearchRequest();
-            searchRequest.indices("test");
-            searchRequest.types("test");
-            searchRequest.source(builder);
-            SearchResponse searchResponse = client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            long hits = searchResponse.getHits().getTotalHits();
-            assertEquals(1, hits);
-            logger.info("hits = {} source = {}", hits, searchResponse.getHits().getHits()[0].getSourceAsString());
-            assertEquals("{\"test\":\"Hello Jörg\"}", searchResponse.getHits().getHits()[0].getSourceAsString());
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        }
-    }
-}

elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java

@@ -1,10 +0,0 @@
-package org.xbib.elx.node;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
-    @Override
-    public ExtendedNodeClient getExtendedClient() {
-        return new ExtendedNodeClient();
-    }
-}

elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java

@@ -8,15 +8,15 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.Plugin;
-import org.xbib.elx.common.AbstractExtendedClient;
+import org.xbib.elx.common.AbstractAdminClient;
 
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 
-public class ExtendedNodeClient extends AbstractExtendedClient {
+public class NodeAdminClient extends AbstractAdminClient {
 
-    private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
+    private static final Logger logger = LogManager.getLogger(NodeAdminClient.class.getName());
 
     private Node node;
 

elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.node;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
+
+    @Override
+    public NodeAdminClient getClient() {
+        return new NodeAdminClient();
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java

@@ -0,0 +1,68 @@
+package org.xbib.elx.node;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.Plugin;
+import org.xbib.elx.common.AbstractBulkClient;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+public class NodeBulkClient extends AbstractBulkClient {
+
+    private static final Logger logger = LogManager.getLogger(NodeBulkClient.class.getName());
+
+    private Node node;
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        if (settings == null) {
+            return null;
+        }
+        String version = System.getProperty("os.name")
+                + " " + System.getProperty("java.vm.name")
+                + " " + System.getProperty("java.vm.vendor")
+                + " " + System.getProperty("java.runtime.version")
+                + " " + System.getProperty("java.vm.version");
+        Settings effectiveSettings = Settings.builder().put(settings)
+                .put("node.client", true)
+                .put("node.master", false)
+                .put("node.data", false)
+                .build();
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+        logger.info("creating node client on {} with effective settings {}",
+                version, Strings.toString(builder));
+        Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
+        this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
+        try {
+            node.start();
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+        return node.client();
+    }
+
+    @Override
+    protected void closeClient() throws IOException {
+        if (node != null) {
+            logger.debug("closing node client");
+            node.close();
+        }
+    }
+
+    private static class BulkNode extends Node {
+
+        BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
+            super(env, classpathPlugins);
+        }
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.node;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
+
+    @Override
+    public NodeBulkClient getClient() {
+        return new NodeBulkClient();
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java

@@ -0,0 +1,68 @@
+package org.xbib.elx.node;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.plugins.Plugin;
+import org.xbib.elx.common.AbstractSearchClient;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+public class NodeSearchClient extends AbstractSearchClient {
+
+    private static final Logger logger = LogManager.getLogger(NodeSearchClient.class.getName());
+
+    private Node node;
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        if (settings == null) {
+            return null;
+        }
+        String version = System.getProperty("os.name")
+                + " " + System.getProperty("java.vm.name")
+                + " " + System.getProperty("java.vm.vendor")
+                + " " + System.getProperty("java.runtime.version")
+                + " " + System.getProperty("java.vm.version");
+        Settings effectiveSettings = Settings.builder().put(settings)
+                .put("node.client", true)
+                .put("node.master", false)
+                .put("node.data", false)
+                .build();
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
+        logger.info("creating node client on {} with effective settings {}",
+                version, Strings.toString(builder));
+        Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
+        this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
+        try {
+            node.start();
+        } catch (Exception e) {
+            throw new IOException(e);
+        }
+        return node.client();
+    }
+
+    @Override
+    protected void closeClient() throws IOException {
+        if (node != null) {
+            logger.debug("closing node client");
+            node.close();
+        }
+    }
+
+    private static class BulkNode extends Node {
+
+        BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
+            super(env, classpathPlugins);
+        }
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.node;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
+
+    @Override
+    public NodeSearchClient getClient() {
+        return new NodeSearchClient();
+    }
+}

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.node.NodeAdminClientProvider

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.node.NodeBulkClientProvider

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.node.ExtendedNodeClientProvider

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.node.NodeSearchClientProvider

elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java

@@ -0,0 +1,174 @@
+package org.xbib.elx.node.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ExtendWith(TestExtension.class)
+class BulkClientTest {
+
+    private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
+
+    private static final Long ACTIONS = 10000L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
+
+    private final TestExtension.Helper helper;
+
+    BulkClientTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testSingleDoc() throws Exception {
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testNewIndex() throws Exception {
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
+                .build();
+        bulkClient.newIndex("test");
+        bulkClient.close();
+    }
+
+    @Test
+    void testMapping() throws Exception {
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .build();
+        XContentBuilder builder = JsonXContent.contentBuilder()
+                .startObject()
+                .startObject("doc")
+                .startObject("properties")
+                .startObject("location")
+                .field("type", "geo_point")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        bulkClient.newIndex("test", Settings.EMPTY, builder);
+        assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
+        bulkClient.close();
+    }
+
+    @Test
+    void testRandomDocs() throws Exception {
+        long numactions = ACTIONS;
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                bulkClient.index("test", null, false,
+                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(numactions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testThreadedRandomDocs() throws Exception {
+        int maxthreads = Runtime.getRuntime().availableProcessors();
+        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
+        final long actions = ACTIONS;
+        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            Settings settings = Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .build();
+            bulkClient.newIndex("test", settings);
+            bulkClient.startBulk("test", 0, 1000);
+            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+            final CountDownLatch latch = new CountDownLatch(maxthreads);
+            for (int i = 0; i < maxthreads; i++) {
+                executorService.execute(() -> {
+                    for (int i1 = 0; i1 < actions; i1++) {
+                        bulkClient.index("test", null, false,
+                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                    }
+                    latch.countDown();
+                });
+            }
+            logger.info("waiting for latch...");
+            if (latch.await(30L, TimeUnit.SECONDS)) {
+                logger.info("flush...");
+                bulkClient.flush();
+                bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+                logger.info("got all responses, executor service shutdown...");
+                executorService.shutdown();
+                executorService.awaitTermination(30L, TimeUnit.SECONDS);
+                logger.info("pool is shut down");
+            } else {
+                logger.warn("latch timeout");
+            }
+            bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
+            assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        } finally {
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+}

elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java

@@ -2,17 +2,12 @@ package org.xbib.elx.node.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.common.Parameters;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -25,9 +20,9 @@ class DuplicateIDTest {
 
     private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
 
-    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
+    private static final Long ACTIONS = 100L;
 
-    private static final Long ACTIONS = 50L;
+    private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
 
     private final TestExtension.Helper helper;
 
@@ -38,23 +33,24 @@ class DuplicateIDTest {
     @Test
     void testDuplicateDocIDs() throws Exception {
         long numactions = ACTIONS;
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
                 .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
                 .build();
         try {
-            client.newIndex("test_dup");
+            client.newIndex("test");
             for (int i = 0; i < ACTIONS; i++) {
-                client.index("test_dup", helper.randomString(1), false,
+                client.index("test", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
             client.flush();
             client.waitForResponses(30L, TimeUnit.SECONDS);
-            client.refreshIndex("test_dup");
+            client.refreshIndex("test");
             SearchSourceBuilder builder = new SearchSourceBuilder();
             builder.query(QueryBuilders.matchAllQuery());
             SearchRequest searchRequest = new SearchRequest();
-            searchRequest.indices("test_dup");
+            searchRequest.indices("test");
+            searchRequest.types("test");
             searchRequest.source(builder);
             long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
             logger.info("hits = {}", hits);
@@ -63,11 +59,11 @@ class DuplicateIDTest {
             logger.warn("skipping, no node available");
         } finally {
             client.close();
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
+            assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
             if (client.getBulkController().getLastBulkError() != null) {
                 logger.error("error", client.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java

@@ -2,17 +2,15 @@ package org.xbib.elx.node.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexPruneResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,24 +35,27 @@ class IndexPruneTest {
 
     @Test
     void testPrune() throws IOException {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
+                .build();
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
                 .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test_prune1", settings);
-            client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
-            client.newIndex("test_prune2", settings);
-            client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
-            client.newIndex("test_prune3", settings);
-            client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
-            client.newIndex("test_prune4", settings);
-            client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
+            bulkClient.newIndex("test_prune1", settings);
+            adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
+            bulkClient.newIndex("test_prune2", settings);
+            adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
+            bulkClient.newIndex("test_prune3", settings);
+            adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
+            bulkClient.newIndex("test_prune4", settings);
+            adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
             IndexPruneResult indexPruneResult =
-                    client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
+                    adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
             assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
             assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
@@ -62,7 +63,7 @@ class IndexPruneTest {
             List<Boolean> list = new ArrayList<>();
             for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
                 IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
-                indicesExistsRequest.indices(new String[] { index });
+                indicesExistsRequest.indices(index);
                 IndicesExistsResponse indicesExistsResponse =
                         client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
                 list.add(indicesExistsResponse.isExists());
@@ -72,14 +73,13 @@ class IndexPruneTest {
             assertFalse(list.get(1));
             assertTrue(list.get(2));
             assertTrue(list.get(3));
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            adminClient.close();
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java

@@ -3,16 +3,16 @@ package org.xbib.elx.node.test;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexShiftResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -34,52 +34,55 @@ class IndexShiftTest {
 
     @Test
     void testIndexShift() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
+                .build();
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
                 .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test1234", settings);
+            client.newIndex("test_shift", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test1234", helper.randomString(1), false,
+                client.index("test_shift", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
             IndexShiftResult indexShiftResult =
-                    client.shiftIndex("test_shift", "test1234", Arrays.asList("a", "b", "c"));
+                    client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
             assertTrue(indexShiftResult.getNewAliases().contains("a"));
             assertTrue(indexShiftResult.getNewAliases().contains("b"));
             assertTrue(indexShiftResult.getNewAliases().contains("c"));
             assertTrue(indexShiftResult.getMovedAliases().isEmpty());
 
-            Map<String, String> aliases = client.getAliases("test1234");
+            Map<String, String> aliases = client.getAliases("test_shift");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
-            assertTrue(aliases.containsKey("test_shift"));
+            assertTrue(aliases.containsKey("test"));
 
-            String resolved = client.resolveAlias("test_shift");
+            String resolved = client.resolveAlias("test");
             aliases = client.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
-            assertTrue(aliases.containsKey("test_shift"));
+            assertTrue(aliases.containsKey("test"));
 
-            client.newIndex("test5678", settings);
+            client.newIndex("test_shift2", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test5678", helper.randomString(1), false,
+                client.index("test_shift2", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
             client.flush();
             client.waitForResponses(30L, TimeUnit.SECONDS);
 
-            indexShiftResult = client.shiftIndex("test_shift", "test5678", Arrays.asList("d", "e", "f"),
-                    (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                            index, alias).filter(QueryBuilders.termQuery("my_key", alias)))
+            indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
+                    (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                            .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
             );
             assertTrue(indexShiftResult.getNewAliases().contains("d"));
             assertTrue(indexShiftResult.getNewAliases().contains("e"));
@@ -88,7 +91,7 @@ class IndexShiftTest {
             assertTrue(indexShiftResult.getMovedAliases().contains("b"));
             assertTrue(indexShiftResult.getMovedAliases().contains("c"));
 
-            aliases = client.getAliases("test5678");
+            aliases = client.getAliases("test_shift2");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
@@ -96,7 +99,7 @@ class IndexShiftTest {
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
 
-            resolved = client.resolveAlias("test_shift");
+            resolved = client.resolveAlias("test");
             aliases = client.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
@@ -104,15 +107,13 @@ class IndexShiftTest {
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            adminClient.close();
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java

@@ -2,14 +2,15 @@ package org.xbib.elx.node.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.api.IndexDefinition;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -29,41 +30,56 @@ class SmokeTest {
 
     @Test
     void smokeTest() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
                 .build();
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .build();
+        IndexDefinition indexDefinition =
+                adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
         try {
+            assertEquals(helper.getClusterName(), client.getClusterName());
             client.newIndex("test_smoke");
             client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
             client.flush();
             client.waitForResponses(30, TimeUnit.SECONDS);
-            assertEquals(helper.getClusterName(), client.getClusterName());
             client.checkMapping("test_smoke");
             client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
+            client.delete("test_smoke", "1");
             client.flush();
+            client.waitForResponses(30, TimeUnit.SECONDS);
             client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
+            client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
             client.delete("test_smoke", "1");
+            client.flush();
             client.deleteIndex("test_smoke");
-            IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke_2", Settings.settingsBuilder()
+            IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
                     .build());
             assertEquals(0, indexDefinition.getReplicaLevel());
             client.newIndex(indexDefinition);
             client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
             client.flush();
+            client.waitForResponses(30, TimeUnit.SECONDS);
             client.updateReplicaLevel(indexDefinition, 2);
             int replica = client.getReplicaLevel(indexDefinition);
             assertEquals(2, replica);
             client.deleteIndex(indexDefinition);
-            assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
-            assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount());
+            //assertEquals(0, client.getBulkMetric().getFailed().getCount());
+            //assertEquals(6, client.getBulkMetric().getSucceeded().getCount());
         } catch (NoNodeAvailableException e) {
             logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
+            assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            // close admin after bulk
+            adminClient.deleteIndex(indexDefinition);
+            adminClient.close();
         }
     }
 }

elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClient.java

@@ -1,134 +0,0 @@
-package org.xbib.elx.transport;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.cluster.node.DiscoveryNodes;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.jboss.netty.channel.DefaultChannelFuture;
-import org.xbib.elx.common.AbstractExtendedClient;
-import org.xbib.elx.common.util.NetworkUtils;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * Transport client with additional methods using the BulkProcessor.
- */
-public class ExtendedTransportClient extends AbstractExtendedClient {
-
-    private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
-
-    @Override
-    protected ElasticsearchClient createClient(Settings settings) {
-        if (settings != null) {
-            String systemIdentifier = System.getProperty("os.name")
-                    + " " + System.getProperty("java.vm.name")
-                    + " " + System.getProperty("java.vm.vendor")
-                    + " " + System.getProperty("java.vm.version")
-                    + " Elasticsearch " + Version.CURRENT.toString();
-            Settings effectiveSettings = Settings.builder()
-                    // for thread pool size
-                    .put("processors",
-                            settings.getAsInt("processors", Runtime.getRuntime().availableProcessors()))
-                    .put("client.transport.sniff", false) // do not sniff
-                    .put("client.transport.nodes_sampler_interval", "1m") // do not ping
-                    .put("client.transport.ping_timeout", "1m") // wait for unresponsive nodes a very long time before disconnect
-                    .put("client.transport.ignore_cluster_name", true) // connect to any cluster
-                    // custom settings may override defaults
-                    .put(settings)
-                    .build();
-            logger.info("creating transport client on {} with custom settings {} and effective settings {}",
-                    systemIdentifier, settings.getAsMap(), effectiveSettings.getAsMap());
-            // we need to disable dead lock check because we may have mixed node/transport clients
-            DefaultChannelFuture.setUseDeadLockChecker(false);
-            return TransportClient.builder().settings(effectiveSettings).build();
-        }
-        return null;
-    }
-
-    @Override
-    protected void closeClient() {
-        if (getClient() != null) {
-            TransportClient client = (TransportClient) getClient();
-            client.close();
-            client.threadPool().shutdown();
-        }
-    }
-
-    @Override
-    public ExtendedTransportClient init(Settings settings) throws IOException {
-        super.init(settings);
-        // additional auto-connect
-        try {
-            Collection<InetSocketTransportAddress> addrs = findAddresses(settings);
-            if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
-                throw new NoNodeAvailableException("no cluster nodes available, check settings "
-                        + settings.toString());
-            }
-        } catch (IOException e) {
-            logger.error(e.getMessage(), e);
-        }
-        return this;
-    }
-
-    private Collection<InetSocketTransportAddress> findAddresses(Settings settings) throws IOException {
-        final int defaultPort = settings.getAsInt("port", 9300);
-        Collection<InetSocketTransportAddress> addresses = new ArrayList<>();
-        for (String hostname : settings.getAsArray("host")) {
-            String[] splitHost = hostname.split(":", 2);
-            if (splitHost.length == 2) {
-                try {
-                    String host = splitHost[0];
-                    InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
-                    int port = Integer.parseInt(splitHost[1]);
-                    InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, port);
-                    addresses.add(address);
-                } catch (NumberFormatException e) {
-                    logger.warn(e.getMessage(), e);
-                }
-            }
-            if (splitHost.length == 1) {
-                String host = splitHost[0];
-                InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
-                InetSocketTransportAddress address = new InetSocketTransportAddress(inetAddress, defaultPort);
-                addresses.add(address);
-            }
-        }
-        return addresses;
-    }
-
-    private boolean connect(Collection<InetSocketTransportAddress> addresses, boolean autodiscover) {
-        if (getClient() == null) {
-            throw new IllegalStateException("no client present");
-        }
-        logger.debug("trying to connect to {}", addresses);
-        TransportClient transportClient = (TransportClient) getClient();
-        transportClient.addTransportAddresses(addresses);
-        List<DiscoveryNode> nodes = transportClient.connectedNodes();
-        logger.info("connected to nodes = {}", nodes);
-        if (nodes != null && !nodes.isEmpty()) {
-            if (autodiscover) {
-                logger.debug("trying to auto-discover all nodes...");
-                ClusterStateRequestBuilder clusterStateRequestBuilder =
-                        new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
-                ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
-                DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
-                transportClient.addDiscoveryNodes(discoveryNodes);
-                logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes());
-            }
-            return true;
-        }
-        return false;
-    }
-}

elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java

@@ -1,11 +0,0 @@
-package org.xbib.elx.transport;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
-
-    @Override
-    public ExtendedTransportClient getExtendedClient() {
-        return new ExtendedTransportClient();
-    }
-}

elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java

@@ -0,0 +1,40 @@
+package org.xbib.elx.transport;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractAdminClient;
+
+import java.io.IOException;
+
+/**
+ * Transport admin client.
+ */
+public class TransportAdminClient extends AbstractAdminClient {
+
+    private final TransportClientHelper helper;
+
+    public TransportAdminClient() {
+        this.helper = new TransportClientHelper();
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    protected void closeClient() {
+        if (getClient() != null) {
+            TransportClient client = (TransportClient) getClient();
+            client.close();
+            client.threadPool().shutdown();
+        }
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init((TransportClient) getClient(), settings);
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.transport;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
+
+    @Override
+    public TransportAdminClient getClient() {
+        return new TransportAdminClient();
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java

@@ -0,0 +1,39 @@
+package org.xbib.elx.transport;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractBulkClient;
+import java.io.IOException;
+
+/**
+ * Transport search client with additional methods.
+ */
+public class TransportBulkClient extends AbstractBulkClient {
+
+    private final TransportClientHelper helper;
+
+    public TransportBulkClient() {
+        this.helper = new TransportClientHelper();
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    protected void closeClient() {
+        if (getClient() != null) {
+            TransportClient client = (TransportClient) getClient();
+            client.close();
+            client.threadPool().shutdown();
+        }
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init((TransportClient) getClient(), settings);
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.transport;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
+
+    @Override
+    public TransportBulkClient getClient() {
+        return new TransportBulkClient();
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java

@@ -0,0 +1,161 @@
+package org.xbib.elx.transport;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.transport.netty4.Netty4Plugin;
+import org.xbib.elx.common.util.NetworkUtils;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+public class TransportClientHelper {
+
+
+    private static final Logger logger = LogManager.getLogger(TransportAdminClient.class.getName());
+
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        if (settings != null) {
+            String systemIdentifier = System.getProperty("os.name")
+                    + " " + System.getProperty("java.vm.name")
+                    + " " + System.getProperty("java.vm.vendor")
+                    + " " + System.getProperty("java.vm.version")
+                    + " Elasticsearch " + Version.CURRENT.toString();
+            Settings transportClientSettings = getTransportClientSettings(settings);
+            //XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
+            XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
+            logger.log(Level.INFO, "creating transport client on {} with settings {}",
+                    systemIdentifier,
+                    //Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
+                    Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
+                            ToXContent.EMPTY_PARAMS).endObject()));
+            return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
+        }
+        return null;
+    }
+
+    public void init(TransportClient transportClient, Settings settings) throws IOException {
+        Collection<TransportAddress> addrs = findAddresses(settings);
+        if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
+            throw new NoNodeAvailableException("no cluster nodes available, check settings "
+                    + settings.toString());
+        }
+    }
+
+    @Override
+    public ExtendedTransportClient init(Settings settings) throws IOException {
+        super.init(settings);
+        // additional auto-connect
+        try {
+            Collection<TransportAddress> addrs = findAddresses(settings);
+            if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
+                throw new NoNodeAvailableException("no cluster nodes available, check settings "
+                        + settings.toString());
+            }
+        } catch (IOException e) {
+            logger.error(e.getMessage(), e);
+        }
+        return this;
+    }
+
+    private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
+        final int defaultPort = settings.getAsInt("port", 9300);
+        Collection<TransportAddress> addresses = new ArrayList<>();
+        for (String hostname : settings.getAsList("host")) {
+            String[] splitHost = hostname.split(":", 2);
+            if (splitHost.length == 2) {
+                try {
+                    String host = splitHost[0];
+                    InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
+                    int port = Integer.parseInt(splitHost[1]);
+                    TransportAddress address = new TransportAddress(inetAddress, port);
+                    addresses.add(address);
+                } catch (NumberFormatException e) {
+                    logger.warn(e.getMessage(), e);
+                }
+            }
+            if (splitHost.length == 1) {
+                String host = splitHost[0];
+                InetAddress inetAddress = NetworkUtils.resolveInetAddress(host, null);
+                TransportAddress address = new TransportAddress(inetAddress, defaultPort);
+                addresses.add(address);
+            }
+        }
+        return addresses;
+    }
+
+    private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) {
+        if (getClient() == null) {
+            throw new IllegalStateException("no client present");
+        }
+        TransportClient transportClient = (TransportClient) getClient();
+        for (TransportAddress address : addresses) {
+            transportClient.addTransportAddresses(address);
+        }
+        List<DiscoveryNode> nodes = transportClient.connectedNodes();
+        logger.info("connected to nodes = {}", nodes);
+        if (nodes != null && !nodes.isEmpty()) {
+            if (autodiscover) {
+                logger.debug("trying to auto-discover all nodes...");
+                ClusterStateRequestBuilder clusterStateRequestBuilder =
+                        new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
+                ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
+                DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
+                addDiscoveryNodes(transportClient, discoveryNodes);
+                logger.info("after auto-discovery: connected to {}", transportClient.connectedNodes());
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private Settings getTransportClientSettings(Settings settings) {
+        return Settings.builder()
+                // "cluster.name"
+                .put(ClusterName.CLUSTER_NAME_SETTING.getKey(),
+                        settings.get(ClusterName.CLUSTER_NAME_SETTING.getKey()))
+                // "processors"
+                .put(EsExecutors.PROCESSORS_SETTING.getKey(),
+                        settings.get(EsExecutors.PROCESSORS_SETTING.getKey(),
+                                String.valueOf(Runtime.getRuntime().availableProcessors())))
+                // "transport.type"
+                .put(NetworkModule.TRANSPORT_TYPE_KEY,
+                        Netty4Plugin.NETTY_TRANSPORT_NAME)
+                .build();
+    }
+
+    private void addDiscoveryNodes(TransportClient transportClient, DiscoveryNodes discoveryNodes) {
+        for (DiscoveryNode discoveryNode : discoveryNodes) {
+            transportClient.addTransportAddress(discoveryNode.getAddress());
+        }
+    }
+
+    static class MyTransportClient extends TransportClient {
+
+        MyTransportClient(Settings settings, Collection<Class<? extends Plugin>> plugins) {
+            super(settings, plugins);
+        }
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java

@@ -0,0 +1,39 @@
+package org.xbib.elx.transport;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractSearchClient;
+import java.io.IOException;
+
+/**
+ * Transport search client with additional methods.
+ */
+public class TransportSearchClient extends AbstractSearchClient {
+
+    private final TransportClientHelper helper;
+
+    public TransportSearchClient() {
+        this.helper = new TransportClientHelper();
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    protected void closeClient() {
+        if (getClient() != null) {
+            TransportClient client = (TransportClient) getClient();
+            client.close();
+            client.threadPool().shutdown();
+        }
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init((TransportClient) getClient(), settings);
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.transport;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
+
+    @Override
+    public TransportSearchClient getClient() {
+        return new TransportSearchClient();
+    }
+}

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.transport.TransportAdminClientProvider

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.transport.TransportBulkClientProvider

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.transport.ExtendedTransportClientProvider

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.transport.TransportSearchClientProvider

elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java

@@ -0,0 +1,188 @@
+package org.xbib.elx.transport.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.transport.TransportBulkClient;
+import org.xbib.elx.transport.TransportBulkClientProvider;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ExtendWith(TestExtension.class)
+class BulkClientTest {
+
+    private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
+
+    private static final Long ACTIONS = 10000L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
+
+    private final TestExtension.Helper helper;
+
+    BulkClientTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testSingleDoc() throws Exception {
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testNewIndex() throws Exception {
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
+                .build();
+        bulkClient.newIndex("test");
+        bulkClient.close();
+    }
+
+    @Test
+    void testMapping() throws Exception {
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .build();
+        XContentBuilder builder = JsonXContent.contentBuilder()
+                .startObject()
+                .startObject("doc")
+                .startObject("properties")
+                .startObject("location")
+                .field("type", "geo_point")
+                .endObject()
+                .endObject()
+                .endObject()
+                .endObject();
+        bulkClient.newIndex("test", Settings.EMPTY, builder);
+        assertTrue(bulkClient.getMapping("test", "doc").containsKey("properties"));
+        bulkClient.close();
+    }
+
+    @Test
+    void testRandomDocs() throws Exception {
+        long numactions = ACTIONS;
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                bulkClient.index("test", null, false,
+                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(numactions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testThreadedRandomDocs() throws Exception {
+        int maxthreads = Runtime.getRuntime().availableProcessors();
+        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
+        final long actions = ACTIONS;
+        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
+                .build();
+        try {
+            Settings settings = Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .build();
+            bulkClient.newIndex("test", settings);
+            bulkClient.startBulk("test", 0, 1000);
+            logger.info("index created");
+            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+            final CountDownLatch latch = new CountDownLatch(maxthreads);
+            for (int i = 0; i < maxthreads; i++) {
+                executorService.execute(() -> {
+                    for (int i1 = 0; i1 < actions; i1++) {
+                        bulkClient.index("test", null, false,
+                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                    }
+                    latch.countDown();
+                });
+            }
+            logger.info("waiting for latch...");
+            if (latch.await(30L, TimeUnit.SECONDS)) {
+                logger.info("flush...");
+                bulkClient.flush();
+                bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+                logger.info("got all responses, executor service shutdown...");
+                executorService.shutdown();
+                executorService.awaitTermination(30L, TimeUnit.SECONDS);
+                logger.info("pool is shut down");
+            } else {
+                logger.warn("latch timeout");
+            }
+            bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
+            assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        } catch (NoNodeAvailableException e) {
+            logger.warn("skipping, no node available");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+}

elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java

@@ -1,201 +0,0 @@
-package org.xbib.elx.transport.test;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.util.concurrent.EsExecutors;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.common.Parameters;
-import org.xbib.elx.transport.ExtendedTransportClient;
-import org.xbib.elx.transport.ExtendedTransportClientProvider;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-@ExtendWith(TestExtension.class)
-class ClientTest {
-
-    private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
-
-    private static final Long ACTIONS = 100L;
-
-    private static final Long MAX_ACTIONS_PER_REQUEST = 1000L;
-
-    private final TestExtension.Helper helper;
-
-    ClientTest(TestExtension.Helper helper) {
-        this.helper = helper;
-    }
-
-    @Test
-    void testClientIndexOp() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        client.newIndex("test1");
-        client.close();
-    }
-
-    @Test
-    void testMapping() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        XContentBuilder builder = jsonBuilder()
-                .startObject()
-                .startObject("doc")
-                .startObject("properties")
-                .startObject("location")
-                .field("type", "geo_point")
-                .endObject()
-                .endObject()
-                .endObject()
-                .endObject();
-        client.newIndex("test2", Settings.EMPTY, builder.string());
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices("test2");
-        GetMappingsResponse getMappingsResponse =
-                client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
-        logger.info("mappings={}", getMappingsResponse.getMappings());
-        assertTrue(getMappingsResponse.getMappings().get("test2").containsKey("doc"));
-        client.close();
-    }
-
-    @Test
-    void testSingleDoc() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            client.newIndex("test3");
-            client.index("test3", "1", true, "{ \"name\" : \"Hello World\"}");
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.close();
-        }
-    }
-
-    @Test
-    void testRandomDocs() throws Exception {
-        long numactions = ACTIONS;
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            client.newIndex("test4");
-            for (int i = 0; i < ACTIONS; i++) {
-                client.index("test4", null, false,
-                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-            }
-            client.flush();
-            client.waitForResponses(60L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test4");
-            SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
-                    .setIndices("test4")
-                    .setQuery(QueryBuilders.matchAllQuery())
-                    .setSize(0);
-            assertEquals(numactions,
-                    searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
-            client.close();
-        }
-    }
-
-    @Test
-    void testThreadedRandomDocs() throws Exception {
-        int maxthreads = Runtime.getRuntime().availableProcessors();
-        long maxactions = MAX_ACTIONS_PER_REQUEST;
-        final long maxloop = ACTIONS;
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxactions)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            client.newIndex("test5")
-                    .startBulk("test5", -1, 1000);
-            ThreadPoolExecutor pool = EsExecutors.newFixed("transportclient-test", maxthreads, 30,
-                            EsExecutors.daemonThreadFactory("transportclient-test"));
-            final CountDownLatch latch = new CountDownLatch(maxthreads);
-            for (int i = 0; i < maxthreads; i++) {
-                pool.execute(() -> {
-                    for (int i1 = 0; i1 < maxloop; i1++) {
-                        client.index("test5",null, false,
-                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-                    }
-                    latch.countDown();
-                });
-            }
-            logger.info("waiting for latch...");
-            if (latch.await(60, TimeUnit.SECONDS)) {
-                logger.info("flush ...");
-                client.flush();
-                client.waitForResponses(60L, TimeUnit.SECONDS);
-                logger.info("pool shutdown ...");
-                pool.shutdown();
-                logger.info("poot is shut down");
-            } else {
-                logger.warn("latch timeout");
-            }
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            client.stopBulk("test5", 60L, TimeUnit.SECONDS);
-            assertEquals(maxthreads * maxloop, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test5");
-            SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
-                    .setIndices("test5")
-                    .setQuery(QueryBuilders.matchAllQuery())
-                    .setSize(0);
-            assertEquals(maxthreads * maxloop,
-                    searchRequestBuilder.execute().actionGet().getHits().getTotalHits());
-            client.close();
-        }
-    }
-}

elx-transport/src/test/java/org/xbib/elx/transport/test/DuplicateIDTest.java

@@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -11,8 +12,8 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.common.Parameters;
-import org.xbib.elx.transport.ExtendedTransportClient;
-import org.xbib.elx.transport.ExtendedTransportClientProvider;
+import org.xbib.elx.transport.TransportBulkClient;
+import org.xbib.elx.transport.TransportBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -23,11 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 @ExtendWith(TestExtension.class)
 class DuplicateIDTest {
 
-    private final static Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
+    private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
 
-    private final static Long MAX_ACTIONS_PER_REQUEST = 100L;
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
 
-    private final static Long ACTIONS = 50L;
+    private static final Long ACTIONS = 5L;
 
     private final TestExtension.Helper helper;
 
@@ -38,15 +39,15 @@ class DuplicateIDTest {
     @Test
     void testDuplicateDocIDs() throws Exception {
         long numactions = ACTIONS;
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
                 .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(helper.getTransportSettings())
                 .build();
         try {
-            client.newIndex("test_dup");
+            bulkClient.newIndex("test_dup");
             for (int i = 0; i < ACTIONS; i++) {
-                client.index("test_dup", helper.randomString(1), false,
+                bulkClient.index("test_dup", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
             client.flush();
@@ -54,21 +55,25 @@ class DuplicateIDTest {
             client.refreshIndex("test_dup");
             SearchSourceBuilder builder = new SearchSourceBuilder();
             builder.query(QueryBuilders.matchAllQuery());
+            builder.size(0);
             SearchRequest searchRequest = new SearchRequest();
             searchRequest.indices("test_dup");
+            searchRequest.types("test_dup");
             searchRequest.source(builder);
-            long hits = helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet().getHits().getTotalHits();
+            SearchResponse searchResponse =
+                    helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
+            long hits = searchResponse.getHits().getTotalHits();
             logger.info("hits = {}", hits);
             assertTrue(hits < ACTIONS);
         } catch (NoNodeAvailableException e) {
             logger.warn("skipping, no node available");
         } finally {
             client.close();
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
+            assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
             if (client.getBulkController().getLastBulkError() != null) {
                 logger.error("error", client.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-transport/src/test/java/org/xbib/elx/transport/test/IndexPruneTest.java

@@ -2,17 +2,15 @@ package org.xbib.elx.transport.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexPruneResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.transport.ExtendedTransportClient;
-import org.xbib.elx.transport.ExtendedTransportClientProvider;
+import org.xbib.elx.transport.TransportAdminClient;
+import org.xbib.elx.transport.TransportAdminClientProvider;
+import org.xbib.elx.transport.TransportBulkClient;
+import org.xbib.elx.transport.TransportBulkClientProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,8 +35,12 @@ class IndexPruneTest {
 
     @Test
     void testPrune() throws IOException {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
+        final TransportAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(TransportAdminClientProvider.class)
+                .put(helper.getTransportSettings())
+                .build();
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
                 .put(helper.getTransportSettings())
                 .build();
         try {
@@ -46,27 +48,24 @@ class IndexPruneTest {
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test_prune1", settings);
-            client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
-            client.newIndex("test_prune2", settings);
-            client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
-            client.newIndex("test_prune3", settings);
-            client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
-            client.newIndex("test_prune4", settings);
-            client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
-
+            bulkClient.newIndex("test_prune1", settings);
+            adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
+            bulkClient.newIndex("test_prune2", settings);
+            adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
+            bulkClient.newIndex("test_prune3", settings);
+            adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
+            bulkClient.newIndex("test_prune4", settings);
+            adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
             IndexPruneResult indexPruneResult =
-                    client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
-
+                    adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
             assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
             assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
-
             List<Boolean> list = new ArrayList<>();
             for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
                 IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
-                indicesExistsRequest.indices(new String[] { index });
+                indicesExistsRequest.indices(index);
                 IndicesExistsResponse indicesExistsResponse =
                         client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
                 list.add(indicesExistsResponse.isExists());
@@ -76,14 +75,13 @@ class IndexPruneTest {
             assertFalse(list.get(1));
             assertTrue(list.get(2));
             assertTrue(list.get(3));
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            adminClient.close();
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-transport/src/test/java/org/xbib/elx/transport/test/IndexShiftTest.java

@@ -3,16 +3,16 @@ package org.xbib.elx.transport.test;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.metadata.AliasAction;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexShiftResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.transport.ExtendedTransportClient;
-import org.xbib.elx.transport.ExtendedTransportClientProvider;
+import org.xbib.elx.transport.TransportAdminClient;
+import org.xbib.elx.transport.TransportAdminClientProvider;
+import org.xbib.elx.transport.TransportBulkClient;
+import org.xbib.elx.transport.TransportBulkClientProvider;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -34,54 +34,53 @@ class IndexShiftTest {
 
     @Test
     void testIndexShift() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings()).build();
+        final TransportAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(TransportAdminClientProvider.class)
+                .put(helper.getTransportSettings())
+                .build();
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test1234", settings);
+            client.newIndex("test_shift1234", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test1234", helper.randomString(1), false,
+                client.index("test_shift1234", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
             IndexShiftResult indexShiftResult =
-                    client.shiftIndex("test_shift", "test1234", Arrays.asList("a", "b", "c"));
-
+                    client.shiftIndex("test_shift", "test_shift1234", Arrays.asList("a", "b", "c"));
             assertTrue(indexShiftResult.getNewAliases().contains("a"));
             assertTrue(indexShiftResult.getNewAliases().contains("b"));
             assertTrue(indexShiftResult.getNewAliases().contains("c"));
             assertTrue(indexShiftResult.getMovedAliases().isEmpty());
-
-            Map<String, String> aliases = client.getAliases("test1234");
+            Map<String, String> aliases = client.getAliases("test_shift1234");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("test_shift"));
-
-            String resolved = client.resolveAlias("test_shift");
-            aliases = client.getAliases(resolved);
+            String resolved = adminClient.resolveAlias("test_shift");
+            aliases = adminClient.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("test_shift"));
-
-            client.newIndex("test5678", settings);
+            client.newIndex("test_shift5678", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test5678", helper.randomString(1), false,
+                client.index("test_shift5678", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
             client.flush();
             client.waitForResponses(30L, TimeUnit.SECONDS);
-
-            indexShiftResult = client.shiftIndex("test_shift", "test5678", Arrays.asList("d", "e", "f"),
-                    (request, index, alias) -> request.addAliasAction(new IndicesAliasesRequest.AliasActions(AliasAction.Type.ADD,
-                            index, alias).filter(QueryBuilders.termQuery("my_key", alias)))
+            indexShiftResult = client.shiftIndex("test_shift", "test_shift5678", Arrays.asList("d", "e", "f"),
+                    (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                            .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
             );
             assertTrue(indexShiftResult.getNewAliases().contains("d"));
             assertTrue(indexShiftResult.getNewAliases().contains("e"));
@@ -89,32 +88,28 @@ class IndexShiftTest {
             assertTrue(indexShiftResult.getMovedAliases().contains("a"));
             assertTrue(indexShiftResult.getMovedAliases().contains("b"));
             assertTrue(indexShiftResult.getMovedAliases().contains("c"));
-
-            aliases = client.getAliases("test5678");
+            aliases = client.getAliases("test_shift5678");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-
-            resolved = client.resolveAlias("test_shift");
-            aliases = client.getAliases(resolved);
+            resolved = adminClient.resolveAlias("test_shift");
+            aliases = adminClient.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            adminClient.close();
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-transport/src/test/java/org/xbib/elx/transport/test/SmokeTest.java

@@ -2,14 +2,15 @@ package org.xbib.elx.transport.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexDefinition;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.transport.ExtendedTransportClient;
-import org.xbib.elx.transport.ExtendedTransportClientProvider;
+import org.xbib.elx.transport.TransportAdminClient;
+import org.xbib.elx.transport.TransportAdminClientProvider;
+import org.xbib.elx.transport.TransportBulkClient;
+import org.xbib.elx.transport.TransportBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -17,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 
 @ExtendWith(TestExtension.class)
-class SmokeTest extends TestExtension {
+class SmokeTest {
 
     private static final Logger logger = LogManager.getLogger(SmokeTest.class.getName());
 
@@ -28,44 +29,56 @@ class SmokeTest extends TestExtension {
     }
 
     @Test
-    void testSingleDocNodeClient() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
+    void smokeTest() throws Exception {
+        final TransportAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(TransportAdminClientProvider.class)
                 .put(helper.getTransportSettings())
                 .build();
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .build();
+        IndexDefinition indexDefinition =
+                adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
         try {
+            assertEquals(helper.getClusterName(), client.getClusterName());
             client.newIndex("test_smoke");
             client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
             client.flush();
             client.waitForResponses(30, TimeUnit.SECONDS);
-            assertEquals(helper.getClusterName(), client.getClusterName());
             client.checkMapping("test_smoke");
             client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
-            client.flush();
-            client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
             client.delete("test_smoke", "1");
+            client.flush();
+            client.waitForResponses(30, TimeUnit.SECONDS);
+            client.checkMapping("test_smoke");
             client.deleteIndex("test_smoke");
-            IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke_2", Settings.settingsBuilder()
+            IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
                     .build());
             assertEquals(0, indexDefinition.getReplicaLevel());
             client.newIndex(indexDefinition);
             client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
             client.flush();
+            client.waitForResponses(30, TimeUnit.SECONDS);
             client.updateReplicaLevel(indexDefinition, 2);
             int replica = client.getReplicaLevel(indexDefinition);
             assertEquals(2, replica);
-
             client.deleteIndex(indexDefinition);
-            assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
-            assertEquals(4, client.getBulkController().getBulkMetric().getSucceeded().getCount());
+            assertEquals(0, client.getBulkMetric().getFailed().getCount());
+            assertEquals(4, client.getBulkMetric().getSucceeded().getCount());
         } catch (NoNodeAvailableException e) {
             logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            bulkClient.close();
+            assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
+            assertEquals(4, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            // close admin after bulk
+            adminClient.deleteIndex(indexDefinition);
+            adminClient.close();
         }
     }
 }