update to admin/bulk/search API, add missing HTTP actions get index, search scroll, clear scroll

elx-api/src/main/java/org/xbib/elx/api/AdminClient.java

@@ -0,0 +1,187 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.common.settings.Settings;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Interface for extended managing and indexing methods of an Elasticsearch client.
+ */
+public interface AdminClient extends BasicClient {
+
+    /**
+     * Build index definition from settings.
+     *
+     * @param index the index name
+     * @param settings the settings for the index
+     * @return index definition
+     * @throws IOException if settings/mapping URL is invalid/malformed
+     */
+    IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
+
+    Map<String, ?> getMapping(String index) throws IOException;
+
+    Map<String, ?> getMapping(String index, String type) throws IOException;
+
+    /**
+     * Delete an index.
+     * @param indexDefinition the index definition
+     * @return this
+     */
+    AdminClient deleteIndex(IndexDefinition indexDefinition);
+
+    /**
+     * Delete an index.
+     *
+     * @param index index
+     * @return this
+     */
+    AdminClient deleteIndex(String index);
+
+
+    /**
+     * Update replica level.
+     * @param indexDefinition the index definition
+     * @param level the replica level
+     * @return this
+     * @throws IOException if replica setting could not be updated
+     */
+    AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
+
+    /**
+     * Update replica level.
+     *
+     * @param index index
+     * @param level the replica level
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return this
+     * @throws IOException if replica setting could not be updated
+     */
+    AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
+
+    /**
+     * Get replica level.
+     * @param indexDefinition the index name
+     * @return the replica level of the index
+     */
+    int getReplicaLevel(IndexDefinition indexDefinition);
+
+    /**
+     * Get replica level.
+     * @param index the index name
+     * @return the replica level of the index
+     */
+    int getReplicaLevel(String index);
+
+    /**
+     * Force segment merge of an index.
+     * @param indexDefinition the index definition
+     * @return this
+     */
+    boolean forceMerge(IndexDefinition indexDefinition);
+
+    /**
+     * Force segment merge of an index.
+     * @param index the index
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return this
+     */
+    boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
+
+    /**
+     * Resolve alias.
+     *
+     * @param alias the alias
+     * @return this index name behind the alias or the alias if there is no index
+     */
+    String resolveAlias(String alias);
+
+    /**
+     * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
+     * name.
+     *
+     * @param alias the alias
+     * @return the most recent index name pointing to the alias
+     */
+    String resolveMostRecentIndex(String alias);
+
+    /**
+     * Get all index aliases.
+     * @param index the index
+     * @return map of index aliases
+     */
+    Map<String, String> getAliases(String index);
+
+    /**
+     *  Shift from one index to another.
+     * @param indexDefinition the index definition
+     * @param additionalAliases new aliases
+     * @return this
+     */
+    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
+
+    /**
+     * Shift from one index to another.
+     * @param indexDefinition the index definition
+     * @param additionalAliases new aliases
+     * @param indexAliasAdder method to add aliases
+     * @return this
+     */
+    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
+                                IndexAliasAdder indexAliasAdder);
+
+    /**
+     * Shift from one index to another.
+     * @param index         the index name
+     * @param fullIndexName the index name with timestamp
+     * @param additionalAliases  a list of names that should be set as index aliases
+     * @return this
+     */
+    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
+
+    /**
+     * Shift from one index to another.
+     * @param index         the index name
+     * @param fullIndexName the index name with timestamp
+     * @param additionalAliases  a list of names that should be set as index aliases
+     * @param adder         an adder method to create alias term queries
+     * @return this
+     */
+    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
+                                IndexAliasAdder adder);
+
+    /**
+     * Prune index.
+     * @param indexDefinition the index definition
+     * @return the index prune result
+     */
+    IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
+
+    /**
+     * Apply retention policy to prune indices. All indices before delta should be deleted,
+     * but the number of mintokeep indices must be kept.
+     *
+     * @param index         index name
+     * @param fullIndexName index name with timestamp
+     * @param delta timestamp delta (for index timestamps)
+     * @param mintokeep     minimum number of indices to keep
+     * @param perform true if pruning should be executed, false if not
+     * @return the index prune result
+     */
+    IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
+
+    /**
+     * Find the timestamp of the most recently indexed document in the index.
+     *
+     * @param index the index name
+     * @param timestampfieldname the timestamp field name
+     * @return millis UTC millis of the most recent document
+     * @throws IOException if most rcent document can not be found
+     */
+    Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
+}

elx-api/src/main/java/org/xbib/elx/api/AdminClientProvider.java

@@ -0,0 +1,7 @@
+package org.xbib.elx.api;
+
+@FunctionalInterface
+public interface AdminClientProvider<C extends AdminClient> {
+
+    C getClient();
+}

elx-api/src/main/java/org/xbib/elx/api/BasicClient.java

@@ -0,0 +1,62 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+public interface BasicClient extends Closeable {
+
+    /**
+     * Set an Elasticsearch client to extend from it. May be null for TransportClient.
+     * @param client the Elasticsearch client
+     */
+    void setClient(ElasticsearchClient client);
+
+    /**
+     * Return Elasticsearch client.
+     *
+     * @return Elasticsearch client
+     */
+    ElasticsearchClient getClient();
+
+    /**
+     * Initiative the extended client, the bulk metric and bulk controller,
+     * creates instances and connect to cluster, if required.
+     *
+     * @param settings settings
+     * @throws IOException if init fails
+     */
+    void init(Settings settings) throws IOException;
+
+    /**
+     * Get cluster name.
+     * @return the cluster name
+     */
+    String getClusterName();
+
+    /**
+     * Get current health color.
+     *
+     * @param maxWaitTime maximum wait time
+     * @param timeUnit time unit
+     * @return the cluster health color
+     */
+    String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
+
+    /**
+     * Wait for cluster being healthy.
+     *
+     * @param healthColor cluster health color to wait for
+     * @param maxWaitTime   time value
+     * @param timeUnit time unit
+     */
+    void waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
+
+    void waitForShards(long maxWaitTime, TimeUnit timeUnit);
+
+    long getSearchableDocs(String index);
+
+    boolean isIndexExists(String index);
+}

elx-api/src/main/java/org/xbib/elx/api/BulkClient.java

@@ -0,0 +1,227 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import java.io.Flushable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public interface BulkClient extends BasicClient, Flushable {
+
+    /**
+     * Get bulk metric.
+     * @return the bulk metric
+     */
+    BulkMetric getBulkMetric();
+
+    /**
+     * Get buulk control.
+     * @return the bulk control
+     */
+    BulkController getBulkController();
+
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @throws IOException if new index creation fails
+     */
+    void newIndex(String index) throws IOException;
+
+    /**
+     * Create a new index.
+     * @param indexDefinition the index definition
+     * @throws IOException if settings/mapping is invalid or index creation fails
+     */
+    void newIndex(IndexDefinition indexDefinition) throws IOException;
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @param settings settings
+     * @throws IOException if settings is invalid or index creation fails
+     */
+    void newIndex(String index, Settings settings) throws IOException;
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @param settings settings
+     * @param mapping mapping
+     * @throws IOException if settings/mapping is invalid or index creation fails
+     */
+    void newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
+
+    /**
+     * Create a new index.
+     *
+     * @param index index
+     * @param settings settings
+     * @param mapping mapping
+     * @throws IOException if settings/mapping is invalid or index creation fails
+     */
+    void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
+
+    /**
+     * Add index request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when limits are exceeded.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param create true if document must be created
+     * @param source the source
+     * @return this
+     */
+    BulkClient index(String index, String id, boolean create, BytesReference source);
+
+    /**
+     * Index request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when limits are exceeded.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param create true if document is to be created, false otherwise
+     * @param source the source
+     * @return this client methods
+     */
+    BulkClient index(String index, String id, boolean create, String source);
+
+    /**
+     * Index request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     *
+     * @param indexRequest the index request to add
+     * @return this
+     */
+    BulkClient index(IndexRequest indexRequest);
+
+    /**
+     * Delete request.
+     *
+     * @param index the index
+     * @param id    the id
+     * @return this
+     */
+    BulkClient delete(String index, String id);
+
+    /**
+     * Delete request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     *
+     * @param deleteRequest the delete request to add
+     * @return this
+     */
+    BulkClient delete(DeleteRequest deleteRequest);
+
+    /**
+     * Bulked update request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     * Note that updates only work correctly when all operations between nodes are synchronized.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param source the source
+     * @return this
+     */
+    BulkClient update(String index, String id, BytesReference source);
+
+    /**
+     * Update document. Use with precaution! Does not work in all cases.
+     *
+     * @param index  the index
+     * @param id     the id
+     * @param source the source
+     * @return this
+     */
+    BulkClient update(String index, String id, String source);
+
+    /**
+     * Bulked update request. Each request will be added to a queue for bulking requests.
+     * Submitting request will be done when bulk limits are exceeded.
+     * Note that updates only work correctly when all operations between nodes are synchronized.
+     *
+     * @param updateRequest the update request to add
+     * @return this
+     */
+    BulkClient update(UpdateRequest updateRequest);
+
+    /**
+     * Start bulk mode for indexes.
+     * @param indexDefinition index definition
+     * @throws IOException if bulk could not be started
+     */
+    void startBulk(IndexDefinition indexDefinition) throws IOException;
+
+    /**
+     * Start bulk mode.
+     *
+     * @param index index
+     * @param startRefreshIntervalSeconds refresh interval before bulk
+     * @param stopRefreshIntervalSeconds  refresh interval after bulk
+     * @throws IOException if bulk could not be started
+     */
+    void startBulk(String index, long startRefreshIntervalSeconds,
+                          long stopRefreshIntervalSeconds) throws IOException;
+
+
+    /**
+     * Stop bulk mode.
+     *
+     * @param indexDefinition index definition
+     * @throws IOException if bulk could not be startet
+     */
+    void stopBulk(IndexDefinition indexDefinition) throws IOException;
+
+    /**
+     * Stops bulk mode.
+     *
+     * @param index index
+     * @param timeout maximum wait time
+     * @param timeUnit time unit for timeout
+     * @throws IOException if bulk could not be stopped
+     */
+    void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
+
+    /**
+     * Wait for all outstanding bulk responses.
+     *
+     * @param timeout maximum wait time
+     * @param timeUnit unit of timeout value
+     * @return true if wait succeeded, false if wait timed out
+     */
+    boolean waitForResponses(long timeout, TimeUnit timeUnit);
+
+    /**
+     * Update index setting.
+     * @param index the index
+     * @param key the key of the value to be updated
+     * @param value the new value
+     * @param timeout timeout
+     * @param timeUnit time unit
+     * @throws IOException if update index setting failed
+     */
+    void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
+
+    /**
+     * Refresh the index.
+     *
+     * @param index index
+     */
+    void refreshIndex(String index);
+
+    /**
+     * Flush the index. The cluster clears cache and completes indexing.
+     *
+     * @param index index
+     */
+    void flushIndex(String index);
+
+}

elx-api/src/main/java/org/xbib/elx/api/BulkClientProvider.java

@@ -0,0 +1,7 @@
+package org.xbib.elx.api;
+
+@FunctionalInterface
+public interface BulkClientProvider<C extends BulkClient> {
+
+    C getClient();
+}

elx-api/src/main/java/org/xbib/elx/api/BulkMetric.java

@@ -1,5 +1,6 @@
 package org.xbib.elx.api;
 
+import org.elasticsearch.common.settings.Settings;
 import org.xbib.metrics.api.Count;
 import org.xbib.metrics.api.Metered;
 
@@ -7,6 +8,8 @@ import java.io.Closeable;
 
 public interface BulkMetric extends Closeable {
 
+    void init(Settings settings);
+
     Metered getTotalIngest();
 
     Count getTotalIngestSizeInBytes();

elx-api/src/main/java/org/xbib/elx/api/ExtendedClient.java

@@ -1,487 +0,0 @@
-package org.xbib.elx.api;
-
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.bytes.BytesReference;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-
-import java.io.Closeable;
-import java.io.Flushable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Interface for extended managing and indexing methods of an Elasticsearch client.
- */
-public interface ExtendedClient extends Flushable, Closeable {
-
-    /**
-     * Set an Elasticsearch client to extend from it. May be null for TransportClient.
-     * @param client client
-     * @return this client
-     */
-    ExtendedClient setClient(ElasticsearchClient client);
-
-    /**
-     * Return Elasticsearch client.
-     *
-     * @return Elasticsearch client
-     */
-    ElasticsearchClient getClient();
-
-    /**
-     * Get bulk control.
-     * @return the bulk control
-     */
-    BulkController getBulkController();
-
-    /**
-     * Initiative the extended client, the bulk metric and bulk controller,
-     * creates instances and connect to cluster, if required.
-     *
-     * @param settings settings
-     * @return this client
-     * @throws IOException if init fails
-     */
-    ExtendedClient init(Settings settings) throws IOException;
-
-    /**
-     * Build index definition from settings.
-     *
-     * @param index the index name
-     * @param settings the settings for the index
-     * @return index definition
-     * @throws IOException if settings/mapping URL is invalid/malformed
-     */
-    IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings) throws IOException;
-
-    /**
-     * Add index request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when limits are exceeded.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param create true if document must be created
-     * @param source the source
-     * @return this
-     */
-    ExtendedClient index(String index, String id, boolean create, BytesReference source);
-
-    /**
-     * Index request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when limits are exceeded.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param create true if document is to be created, false otherwise
-     * @param source the source
-     * @return this client methods
-     */
-    ExtendedClient index(String index, String id, boolean create, String source);
-
-    /**
-     * Index request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     *
-     * @param indexRequest the index request to add
-     * @return this
-     */
-    ExtendedClient index(IndexRequest indexRequest);
-
-    /**
-     * Delete request.
-     *
-     * @param index the index
-     * @param id    the id
-     * @return this
-     */
-    ExtendedClient delete(String index, String id);
-
-    /**
-     * Delete request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     *
-     * @param deleteRequest the delete request to add
-     * @return this
-     */
-    ExtendedClient delete(DeleteRequest deleteRequest);
-
-    /**
-     * Bulked update request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     * Note that updates only work correctly when all operations between nodes are synchronized.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param source the source
-     * @return this
-     * @throws IOException if update fails
-     */
-    ExtendedClient update(String index, String id, BytesReference source) throws IOException;
-
-    /**
-     * Update document. Use with precaution! Does not work in all cases.
-     *
-     * @param index  the index
-     * @param id     the id
-     * @param source the source
-     * @return this
-     */
-    ExtendedClient update(String index, String id, String source);
-
-    /**
-     * Bulked update request. Each request will be added to a queue for bulking requests.
-     * Submitting request will be done when bulk limits are exceeded.
-     * Note that updates only work correctly when all operations between nodes are synchronized.
-     *
-     * @param updateRequest the update request to add
-     * @return this
-     */
-    ExtendedClient update(UpdateRequest updateRequest);
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @return this
-     * @throws IOException if new index creation fails
-     */
-    ExtendedClient newIndex(String index) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @return this
-     * @throws IOException if settings is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     *
-     * @param index index
-     * @param settings settings
-     * @param mapping mapping
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException;
-
-    /**
-     * Create a new index.
-     * @param indexDefinition the index definition
-     * @return this
-     * @throws IOException if settings/mapping is invalid or index creation fails
-     */
-    ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException;
-
-    /**
-     * Delete an index.
-     * @param indexDefinition the index definition
-     * @return this
-     */
-    ExtendedClient deleteIndex(IndexDefinition indexDefinition);
-
-    /**
-     * Delete an index.
-     *
-     * @param index index
-     * @return this
-     */
-    ExtendedClient deleteIndex(String index);
-
-    /**
-     * Start bulk mode for indexes.
-     * @param indexDefinition index definition
-     * @return this
-     * @throws IOException if bulk could not be started
-     */
-    ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException;
-
-    /**
-     * Start bulk mode.
-     *
-     * @param index index
-     * @param startRefreshIntervalSeconds refresh interval before bulk
-     * @param stopRefreshIntervalSeconds  refresh interval after bulk
-     * @return this
-     * @throws IOException if bulk could not be started
-     */
-    ExtendedClient startBulk(String index, long startRefreshIntervalSeconds,
-                             long stopRefreshIntervalSeconds) throws IOException;
-
-    /**
-     * Stop bulk mode.
-     *
-     * @param indexDefinition index definition
-     * @return this
-     * @throws IOException if bulk could not be startet
-     */
-    ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException;
-
-    /**
-     * Stops bulk mode.
-     *
-     * @param index index
-     * @param timeout maximum wait time
-     * @param timeUnit time unit for timeout
-     * @return this
-     * @throws IOException if bulk could not be stopped
-     */
-    ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException;
-
-    /**
-     * Update replica level.
-     * @param indexDefinition the index definition
-     * @param level the replica level
-     * @return this
-     * @throws IOException if replica setting could not be updated
-     */
-    ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException;
-
-    /**
-     * Update replica level.
-     *
-     * @param index index
-     * @param level the replica level
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return this
-     * @throws IOException if replica setting could not be updated
-     */
-    ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException;
-
-    /**
-     * Get replica level.
-     * @param indexDefinition the index name
-     * @return the replica level of the index
-     */
-    int getReplicaLevel(IndexDefinition indexDefinition);
-
-    /**
-     * Get replica level.
-     * @param index the index name
-     * @return the replica level of the index
-     */
-    int getReplicaLevel(String index);
-
-    /**
-     * Refresh the index.
-     *
-     * @param index index
-     * @return this
-     */
-    ExtendedClient refreshIndex(String index);
-
-    /**
-     * Flush the index. The cluster clears cache and completes indexing.
-     *
-     * @param index index
-     * @return this
-     */
-    ExtendedClient flushIndex(String index);
-
-    /**
-     * Force segment merge of an index.
-     * @param indexDefinition the index definition
-     * @return this
-     */
-    boolean forceMerge(IndexDefinition indexDefinition);
-
-    /**
-     * Force segment merge of an index.
-     * @param index the index
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return this
-     */
-    boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Wait for all outstanding bulk responses.
-     *
-     * @param timeout maximum wait time
-     * @param timeUnit unit of timeout value
-     * @return true if wait succeeded, false if wait timed out
-     */
-    boolean waitForResponses(long timeout, TimeUnit timeUnit);
-
-    /**
-     * Wait for cluster being healthy.
-     *
-     * @param healthColor cluster health color to wait for
-     * @param maxWaitTime   time value
-     * @param timeUnit time unit
-     * @return true if wait succeeded, false if wait timed out
-     */
-    boolean waitForCluster(String healthColor, long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Get current health color.
-     *
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return the cluster health color
-     */
-    String getHealthColor(long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Wait for index recovery (after replica change).
-     *
-     * @param index index
-     * @param maxWaitTime maximum wait time
-     * @param timeUnit time unit
-     * @return true if wait succeeded, false if wait timed out
-     */
-    boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit);
-
-    /**
-     * Update index setting.
-     * @param index the index
-     * @param key the key of the value to be updated
-     * @param value the new value
-     * @param timeout timeout
-     * @param timeUnit time unit
-     * @throws IOException if update index setting failed
-     */
-    void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException;
-
-    /**
-     * Resolve alias.
-     *
-     * @param alias the alias
-     * @return this index name behind the alias or the alias if there is no index
-     */
-    String resolveAlias(String alias);
-
-    /**
-     * Resolve alias to all connected indices, sort index names with most recent timestamp on top, return this index
-     * name.
-     *
-     * @param alias the alias
-     * @return the most recent index name pointing to the alias
-     */
-    String resolveMostRecentIndex(String alias);
-
-    /**
-     * Get all index aliases.
-     * @param index the index
-     * @return map of index aliases
-     */
-    Map<String, String> getAliases(String index);
-
-    /**
-     *  Shift from one index to another.
-     * @param indexDefinition the index definition
-     * @param additionalAliases new aliases
-     * @return this
-     */
-    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases);
-
-    /**
-     * Shift from one index to another.
-     * @param indexDefinition the index definition
-     * @param additionalAliases new aliases
-     * @param indexAliasAdder method to add aliases
-     * @return this
-     */
-    IndexShiftResult shiftIndex(IndexDefinition indexDefinition, List<String> additionalAliases,
-                                IndexAliasAdder indexAliasAdder);
-
-    /**
-     * Shift from one index to another.
-     * @param index         the index name
-     * @param fullIndexName the index name with timestamp
-     * @param additionalAliases  a list of names that should be set as index aliases
-     * @return this
-     */
-    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases);
-
-    /**
-     * Shift from one index to another.
-     * @param index         the index name
-     * @param fullIndexName the index name with timestamp
-     * @param additionalAliases  a list of names that should be set as index aliases
-     * @param adder         an adder method to create alias term queries
-     * @return this
-     */
-    IndexShiftResult shiftIndex(String index, String fullIndexName, List<String> additionalAliases,
-                                IndexAliasAdder adder);
-
-    /**
-     * Prune index.
-     * @param indexDefinition the index definition
-     * @return the index prune result
-     */
-    IndexPruneResult pruneIndex(IndexDefinition indexDefinition);
-
-    /**
-     * Apply retention policy to prune indices. All indices before delta should be deleted,
-     * but the number of mintokeep indices must be kept.
-     *
-     * @param index         index name
-     * @param fullIndexName index name with timestamp
-     * @param delta timestamp delta (for index timestamps)
-     * @param mintokeep     minimum number of indices to keep
-     * @param perform true if pruning should be executed, false if not
-     * @return the index prune result
-     */
-    IndexPruneResult pruneIndex(String index, String fullIndexName, int delta, int mintokeep, boolean perform);
-
-    /**
-     * Find the timestamp of the most recently indexed document in the index.
-     *
-     * @param index the index name
-     * @param timestampfieldname the timestamp field name
-     * @return millis UTC millis of the most recent document
-     * @throws IOException if most rcent document can not be found
-     */
-    Long mostRecentDocument(String index, String timestampfieldname) throws IOException;
-
-    /**
-     * Get cluster name.
-     * @return the cluster name
-     */
-    String getClusterName();
-}

elx-api/src/main/java/org/xbib/elx/api/ExtendedClientProvider.java

@@ -1,7 +0,0 @@
-package org.xbib.elx.api;
-
-@FunctionalInterface
-public interface ExtendedClientProvider<C extends ExtendedClient> {
-
-    C getExtendedClient();
-}

elx-api/src/main/java/org/xbib/elx/api/IndexDefinition.java

@@ -1,7 +1,5 @@
 package org.xbib.elx.api;
 
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.concurrent.TimeUnit;
 
 public interface IndexDefinition {
@@ -14,17 +12,13 @@ public interface IndexDefinition {
 
     String getFullIndexName();
 
-    IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException;
+    IndexDefinition setSettings(String settings);
 
-    IndexDefinition setSettingsUrl(URL settingsUrl);
+    String getSettings();
 
-    URL getSettingsUrl();
+    IndexDefinition setMappings(String mappings);
 
-    IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException;
-
-    IndexDefinition setMappingsUrl(URL mappingsUrl);
-
-    URL getMappingsUrl();
+    String getMappings();
 
     IndexDefinition setDateTimePattern(String timeWindow);
 

elx-api/src/main/java/org/xbib/elx/api/IndexPruneResult.java

@@ -1,6 +1,6 @@
 package org.xbib.elx.api;
 
-import java.util.List;
+import java.util.Collection;
 
 public interface IndexPruneResult {
 
@@ -8,9 +8,9 @@ public interface IndexPruneResult {
 
     State getState();
 
-    List<String> getCandidateIndices();
+    Collection<String> getCandidateIndices();
 
-    List<String> getDeletedIndices();
+    Collection<String> getDeletedIndices();
 
     boolean isAcknowledged();
 }

elx-api/src/main/java/org/xbib/elx/api/IndexShiftResult.java

@@ -1,10 +1,10 @@
 package org.xbib.elx.api;
 
-import java.util.List;
+import java.util.Collection;
 
 public interface IndexShiftResult {
 
-    List<String> getMovedAliases();
+    Collection<String> getMovedAliases();
 
-    List<String> getNewAliases();
+    Collection<String> getNewAliases();
 }

elx-api/src/main/java/org/xbib/elx/api/ReadClient.java

@@ -1,25 +0,0 @@
-package org.xbib.elx.api;
-
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
-import org.elasticsearch.action.get.MultiGetRequest;
-import org.elasticsearch.action.get.MultiGetResponse;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-
-public interface ReadClient {
-
-    ActionFuture<GetResponse> get(GetRequest getRequest);
-
-    void get(GetRequest request, ActionListener<GetResponse> listener);
-
-    ActionFuture<MultiGetResponse> multiGet(MultiGetRequest request);
-
-    void multiGet(MultiGetRequest request, ActionListener<MultiGetResponse> listener);
-
-    ActionFuture<SearchResponse> search(SearchRequest request);
-
-    void search(SearchRequest request, ActionListener<SearchResponse> listener);
-}

elx-api/src/main/java/org/xbib/elx/api/ReadClientProvider.java

@@ -1,6 +0,0 @@
-package org.xbib.elx.api;
-
-public interface ReadClientProvider<C extends ReadClient> {
-
-    C getReadClient();
-}

elx-api/src/main/java/org/xbib/elx/api/SearchClient.java

@@ -0,0 +1,27 @@
+package org.xbib.elx.api;
+
+import org.elasticsearch.action.get.GetRequestBuilder;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import java.util.Optional;
+import java.util.function.Consumer;
+import java.util.stream.Stream;
+
+public interface SearchClient extends BasicClient {
+
+    Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilder);
+
+    Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilder);
+
+    Optional<SearchResponse> search(Consumer<SearchRequestBuilder> searchRequestBuilder);
+
+    Stream<SearchHit> search(Consumer<SearchRequestBuilder> searchRequestBuilder,
+                             TimeValue scrollTime, int scrollSize);
+
+    Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder);
+}

elx-api/src/main/java/org/xbib/elx/api/SearchClientProvider.java

@@ -0,0 +1,7 @@
+package org.xbib.elx.api;
+
+@FunctionalInterface
+public interface SearchClientProvider<C extends SearchClient> {
+
+    C getClient();
+}

elx-common/src/main/java/org/xbib/elx/common/AbstractAdminClient.java

@@ -2,14 +2,8 @@ package org.xbib.elx.common;
 
 import com.carrotsearch.hppc.cursors.ObjectCursor;
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
-import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.search.TotalHits;
-import org.elasticsearch.ElasticsearchTimeoutException;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
@@ -18,13 +12,8 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesAction;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
-import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
-import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.flush.FlushAction;
-import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeAction;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexAction;
@@ -32,36 +21,31 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
-import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchAction;
 import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
 import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.DeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.elasticsearch.common.xcontent.yaml.YamlXContent;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -69,8 +53,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.search.sort.SortBuilder;
 import org.elasticsearch.search.sort.SortBuilders;
 import org.elasticsearch.search.sort.SortOrder;
-import org.xbib.elx.api.BulkController;
-import org.xbib.elx.api.ExtendedClient;
+import org.xbib.elx.api.AdminClient;
 import org.xbib.elx.api.IndexAliasAdder;
 import org.xbib.elx.api.IndexDefinition;
 import org.xbib.elx.api.IndexPruneResult;
@@ -79,13 +62,15 @@ import org.xbib.elx.api.IndexShiftResult;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.nio.charset.MalformedInputException;
 import java.nio.charset.StandardCharsets;
 import java.time.LocalDate;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
@@ -98,22 +83,21 @@ import java.util.TreeSet;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
-public abstract class AbstractExtendedClient implements ExtendedClient {
+public abstract class AbstractAdminClient extends AbstractBasicClient implements AdminClient {
 
-    private static final Logger logger = LogManager.getLogger(AbstractExtendedClient.class.getName());
+    private static final Logger logger = LogManager.getLogger(AbstractAdminClient.class.getName());
 
-    private ElasticsearchClient client;
-
-    private BulkController bulkController;
-
-    private final AtomicBoolean closed;
+    /**
+     * The one and only index type name used in the extended client.
+     * Notr that all Elasticsearch version < 6.2.0 do not allow a prepending "_".
+     */
+    private static final String TYPE_NAME = "doc";
 
     private static final IndexShiftResult EMPTY_INDEX_SHIFT_RESULT = new IndexShiftResult() {
         @Override
@@ -149,363 +133,54 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         }
     };
 
-    protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
-
-    protected abstract void closeClient() throws IOException;
-
-    protected AbstractExtendedClient() {
-        closed = new AtomicBoolean(false);
+    @Override
+    public Map<String, ?> getMapping(String index) throws IOException {
+        return getMapping(index, TYPE_NAME);
     }
 
     @Override
-    public AbstractExtendedClient setClient(ElasticsearchClient client) {
-        this.client = client;
-        return this;
+    public Map<String, ?> getMapping(String index, String mapping) throws IOException {
+        GetMappingsRequestBuilder getMappingsRequestBuilder = new GetMappingsRequestBuilder(client, GetMappingsAction.INSTANCE)
+                .setIndices(index)
+                .setTypes(mapping);
+        GetMappingsResponse getMappingsResponse = getMappingsRequestBuilder.execute().actionGet();
+        logger.info("get mappings response = {}", getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap());
+        return getMappingsResponse.getMappings().get(index).get(mapping).getSourceAsMap();
     }
 
     @Override
-    public ElasticsearchClient getClient() {
-        return client;
-    }
-
-    @Override
-    public BulkController getBulkController() {
-        return bulkController;
-    }
-
-    @Override
-    public AbstractExtendedClient init(Settings settings) throws IOException {
-        logger.info("initializing with settings = " + settings.toDelimitedString(','));
-        if (client == null) {
-            client = createClient(settings);
-        }
-        if (bulkController == null) {
-            bulkController = new DefaultBulkController(this);
-            bulkController.init(settings);
-        }
-        return this;
-    }
-
-    @Override
-    public void flush() throws IOException {
-        if (bulkController != null) {
-            bulkController.flush();
-        }
-    }
-
-    @Override
-    public void close() throws IOException {
-        ensureClient();
-        if (closed.compareAndSet(false, true)) {
-            if (bulkController != null) {
-                logger.info("closing bulk controller");
-                bulkController.close();
-            }
-            closeClient();
-        }
-    }
-
-    @Override
-    public String getClusterName() {
-        ensureClient();
-        try {
-            ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
-            ClusterStateResponse clusterStateResponse =
-                    client.execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
-            return clusterStateResponse.getClusterName().value();
-        } catch (ElasticsearchTimeoutException e) {
-            logger.warn(e.getMessage(), e);
-            return "TIMEOUT";
-        } catch (NoNodeAvailableException e) {
-            logger.warn(e.getMessage(), e);
-            return "DISCONNECTED";
-        } catch (Exception e) {
-            logger.warn(e.getMessage(), e);
-            return "[" + e.getMessage() + "]";
-        }
-    }
-
-    @Override
-    public ExtendedClient newIndex(IndexDefinition indexDefinition) throws IOException {
-        ensureClient();
-        waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
-        URL indexSettings = indexDefinition.getSettingsUrl();
-        if (indexSettings == null) {
-            logger.warn("warning while creating index '{}', no settings/mappings",
-                    indexDefinition.getFullIndexName());
-            newIndex(indexDefinition.getFullIndexName());
-            return this;
-        }
-        URL indexMappings = indexDefinition.getMappingsUrl();
-        if (indexMappings == null) {
-            logger.warn("warning while creating index '{}', no mappings",
-                    indexDefinition.getFullIndexName());
-            newIndex(indexDefinition.getFullIndexName(), indexSettings.openStream(), null);
-            return this;
-        }
-        try (InputStream indexSettingsInput = indexSettings.openStream();
-             InputStream indexMappingsInput = indexMappings.openStream()) {
-            newIndex(indexDefinition.getFullIndexName(), indexSettingsInput, indexMappingsInput);
-        } catch (IOException e) {
-            if (indexDefinition.ignoreErrors()) {
-                logger.warn(e.getMessage(), e);
-                logger.warn("warning while creating index '{}' with settings at {} and mappings at {}",
-                        indexDefinition.getFullIndexName(), indexSettings, indexMappings);
-            } else {
-                logger.error("error while creating index '{}' with settings at {} and mappings at {}",
-                        indexDefinition.getFullIndexName(), indexSettings, indexMappings);
-                throw new IOException(e);
-            }
-        }
-        return this;
-    }
-
-    @Override
-    public ExtendedClient newIndex(String index) throws IOException {
-        return newIndex(index, Settings.EMPTY, (Map<String, Object>) null);
-    }
-
-    @Override
-    public ExtendedClient newIndex(String index, InputStream settings, InputStream mapping) throws IOException {
-        return newIndex(index,
-                Settings.builder().loadFromStream(".json", settings, true).build(),
-                mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
-                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
-    }
-
-    @Override
-    public ExtendedClient newIndex(String index, Settings settings) throws IOException {
-        return newIndex(index, settings, (Map<String, ?>) null);
-    }
-
-    @Override
-    public ExtendedClient newIndex(String index, Settings settings, String mapping) throws IOException {
-        return newIndex(index, settings,
-                mapping != null ? JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
-                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mapping).mapOrdered() : null);
-    }
-
-    @Override
-    public ExtendedClient newIndex(String index, Settings settings, XContentBuilder mapping) throws IOException {
-        ensureClient();
-        if (index == null) {
-            logger.warn("no index name given to create index");
-            return this;
-        }
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
-        if (settings != null) {
-            createIndexRequest.settings(settings);
-        }
-        if (mapping != null) {
-            createIndexRequest.mapping("_doc", mapping);
-        }
-        CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
-        if (createIndexResponse.isAcknowledged()) {
-            return this;
-        }
-        throw new IllegalStateException("index creation not acknowledged");
-    }
-
-    @Override
-    public ExtendedClient newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
-        ensureClient();
-        if (index == null) {
-            logger.warn("no index name given to create index");
-            return this;
-        }
-        CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
-        if (settings != null) {
-            createIndexRequest.settings(settings);
-        }
-        if (mapping != null) {
-            createIndexRequest.mapping("_doc", mapping);
-        }
-        CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
-        if (createIndexResponse.isAcknowledged()) {
-            return this;
-        }
-        throw new IllegalStateException("index creation not acknowledged");
-    }
-
-    @Override
-    public ExtendedClient deleteIndex(IndexDefinition indexDefinition) {
+    public AdminClient deleteIndex(IndexDefinition indexDefinition) {
         return deleteIndex(indexDefinition.getFullIndexName());
     }
 
     @Override
-    public ExtendedClient deleteIndex(String index) {
-        ensureClient();
+    public AdminClient deleteIndex(String index) {
+        ensureClientIsPresent();
         if (index == null) {
             logger.warn("no index name given to delete index");
             return this;
         }
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index);
         client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
+        waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
+        waitForShards(30L, TimeUnit.SECONDS);
         return this;
     }
 
     @Override
-    public ExtendedClient startBulk(IndexDefinition indexDefinition) throws IOException {
-        startBulk(indexDefinition.getFullIndexName(), -1, 1);
-        return this;
-    }
-
-    @Override
-    public ExtendedClient startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
-            throws IOException {
-        if (bulkController != null) {
-            ensureClient();
-            bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
-        }
-        return this;
-    }
-
-    @Override
-    public ExtendedClient stopBulk(IndexDefinition indexDefinition) throws IOException {
-        if (bulkController != null) {
-            ensureClient();
-            bulkController.stopBulkMode(indexDefinition);
-        }
-        return this;
-    }
-
-    @Override
-    public ExtendedClient stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
-        if (bulkController != null) {
-            ensureClient();
-            bulkController.stopBulkMode(index, timeout, timeUnit);
-        }
-        return this;
-    }
-
-    @Override
-    public ExtendedClient index(String index, String id, boolean create, String source) {
-        return index(new IndexRequest().index(index).id(id).create(create)
-                .source(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
-    }
-
-    @Override
-    public ExtendedClient index(String index, String id, boolean create, BytesReference source) {
-        return index(new IndexRequest().index(index).id(id).create(create)
-                .source(source, XContentType.JSON));
-    }
-
-    @Override
-    public ExtendedClient index(IndexRequest indexRequest) {
-        ensureClient();
-        bulkController.bulkIndex(indexRequest);
-        return this;
-    }
-
-    @Override
-    public ExtendedClient delete(String index, String id) {
-        return delete(new DeleteRequest().index(index).id(id));
-    }
-
-    @Override
-    public ExtendedClient delete(DeleteRequest deleteRequest) {
-        ensureClient();
-        bulkController.bulkDelete(deleteRequest);
-        return this;
-    }
-
-    @Override
-    public ExtendedClient update(String index, String id, BytesReference source) {
-        return update(new UpdateRequest().index(index).id(id)
-                .doc(source, XContentType.JSON));
-    }
-
-    @Override
-    public ExtendedClient update(String index, String id, String source) {
-        return update(new UpdateRequest().index(index).id(id)
-                .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
-    }
-
-    @Override
-    public ExtendedClient update(UpdateRequest updateRequest) {
-        ensureClient();
-        bulkController.bulkUpdate(updateRequest);
-        return this;
-    }
-
-    @Override
-    public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
-        ensureClient();
-        return bulkController.waitForBulkResponses(timeout, timeUnit);
-    }
-
-    @Override
-    public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
-        ensureClient();
-        ensureIndexGiven(index);
-        GetSettingsRequest settingsRequest = new GetSettingsRequest();
-        settingsRequest.indices(index);
-        GetSettingsResponse settingsResponse = client.execute(GetSettingsAction.INSTANCE, settingsRequest).actionGet();
-        int shards = settingsResponse.getIndexToSettings().get(index).getAsInt("index.number_of_shards", -1);
-        if (shards > 0) {
-            TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
-            ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
-                    .indices(index)
-                    .waitForActiveShards(shards)
-                    .timeout(timeout);
-            ClusterHealthResponse healthResponse =
-                    client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
-            if (healthResponse != null && healthResponse.isTimedOut()) {
-                logger.warn("timeout waiting for recovery");
-                return false;
-            }
-        }
-        return true;
-    }
-
-    @Override
-    public boolean waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
-        ensureClient();
-        ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
-        TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
-        ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
-                new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
-        if (healthResponse != null && healthResponse.isTimedOut()) {
-            logger.warn("timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name());
-            return false;
-        }
-        return true;
-    }
-
-    @Override
-    public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
-        ensureClient();
-        try {
-            TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
-            ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
-                    new ClusterHealthRequest().timeout(timeout)).actionGet();
-            ClusterHealthStatus status = healthResponse.getStatus();
-            return status.name();
-        } catch (ElasticsearchTimeoutException e) {
-            logger.warn(e.getMessage(), e);
-            return "TIMEOUT";
-        } catch (NoNodeAvailableException e) {
-            logger.warn(e.getMessage(), e);
-            return "DISCONNECTED";
-        } catch (Exception e) {
-            logger.warn(e.getMessage(), e);
-            return "[" + e.getMessage() + "]";
-        }
-    }
-
-    @Override
-    public ExtendedClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
+    public AdminClient updateReplicaLevel(IndexDefinition indexDefinition, int level) throws IOException {
         return updateReplicaLevel(indexDefinition.getFullIndexName(), level,
                 indexDefinition.getMaxWaitTime(), indexDefinition.getMaxWaitTimeUnit());
     }
 
     @Override
-    public ExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
-        waitForCluster("YELLOW", maxWaitTime, timeUnit); // let cluster settle down from critical operations
-        if (level > 0) {
-            updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
-            waitForRecovery(index, maxWaitTime, timeUnit);
+    public AdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) throws IOException {
+        if (level < 1) {
+            logger.warn("invalid replica level");
+            return this;
         }
+        updateIndexSetting(index, "number_of_replicas", level, maxWaitTime, timeUnit);
+        waitForShards(maxWaitTime, timeUnit);
         return this;
     }
 
@@ -528,30 +203,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         return replica;
     }
 
-    @Override
-    public ExtendedClient flushIndex(String index) {
-        if (index != null) {
-            ensureClient();
-            client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
-        }
-        return this;
-    }
-
-    @Override
-    public ExtendedClient refreshIndex(String index) {
-        if (index != null) {
-            ensureClient();
-            client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
-        }
-        return this;
-    }
-
     @Override
     public String resolveMostRecentIndex(String alias) {
+        ensureClientIsPresent();
         if (alias == null) {
             return null;
         }
-        ensureClient();
         GetAliasesRequest getAliasesRequest = new GetAliasesRequest().aliases(alias);
         GetAliasesResponse getAliasesResponse = client.execute(GetAliasesAction.INSTANCE, getAliasesRequest).actionGet();
         Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
@@ -576,7 +233,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public String resolveAlias(String alias) {
-        ensureClient();
+        ensureClientIsPresent();
         ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
         clusterStateRequest.blocks(false);
         clusterStateRequest.metaData(true);
@@ -618,7 +275,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public IndexShiftResult shiftIndex(String index, String fullIndexName,
                                      List<String> additionalAliases, IndexAliasAdder adder) {
-        ensureClient();
+        ensureClientIsPresent();
         if (index == null) {
             return EMPTY_INDEX_SHIFT_RESULT; // nothing to shift to
         }
@@ -644,9 +301,9 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 String alias = entry.getKey();
                 String filter = entry.getValue();
                 indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
-                        .indices(oldIndex).alias(alias));
+                        .index(oldIndex).alias(alias));
                 if (filter != null) {
-                    indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
+                    indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
                             .index(fullIndexName).alias(alias).filter(filter));
                 } else {
                     indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
@@ -670,7 +327,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 } else {
                     String filter = oldAliasMap.get(additionalAlias);
                     indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.remove()
-                            .indices(oldIndex).alias(additionalAlias));
+                            .index(oldIndex).alias(additionalAlias));
                     if (filter != null) {
                         indicesAliasesRequest.addAliasAction(IndicesAliasesRequest.AliasActions.add()
                                 .index(fullIndexName).alias(additionalAlias).filter(filter));
@@ -686,8 +343,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
             logger.debug("indices alias request = {}", indicesAliasesRequest.getAliasActions().toString());
             AcknowledgedResponse indicesAliasesResponse =
                     client.execute(IndicesAliasesAction.INSTANCE, indicesAliasesRequest).actionGet();
-            logger.debug("response isAcknowledged = {} isFragment = {}",
-                    indicesAliasesResponse.isAcknowledged(), indicesAliasesResponse.isFragment());
+            logger.debug("response isAcknowledged = {}",
+                    indicesAliasesResponse.isAcknowledged());
         }
         return new SuccessIndexShiftResult(moveAliases, newAliases);
     }
@@ -706,11 +363,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         if (index.equals(fullIndexName)) {
             return EMPTY_INDEX_PRUNE_RESULT;
         }
-        ensureClient();
+        ensureClientIsPresent();
         GetIndexRequestBuilder getIndexRequestBuilder = new GetIndexRequestBuilder(client, GetIndexAction.INSTANCE);
         GetIndexResponse getIndexResponse = getIndexRequestBuilder.execute().actionGet();
         Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
-        logger.info("pruneIndex: total of {} indices", getIndexResponse.getIndices().length);
+        logger.info("{} indices", getIndexResponse.getIndices().length);
         List<String> candidateIndices = new ArrayList<>();
         for (String s : getIndexResponse.getIndices()) {
             Matcher m = pattern.matcher(s);
@@ -746,29 +403,20 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest()
                 .indices(indicesToDelete.toArray(s));
         AcknowledgedResponse response = client.execute(DeleteIndexAction.INSTANCE, deleteIndexRequest).actionGet();
-        if( response.isAcknowledged()) {
-            logger.log(Level.INFO, "deletion of {} acknowledged, waiting for GREEN", Arrays.asList(s));
-            waitForCluster("GREEN", 30L, TimeUnit.SECONDS);
         return new SuccessPruneResult(candidateIndices, indicesToDelete, response);
-        } else {
-            logger.log(Level.WARN, "deletion of {} not acknowledged", Arrays.asList(s));
-            return new FailPruneResult(candidateIndices, indicesToDelete, response);
-        }
     }
 
     @Override
     public Long mostRecentDocument(String index, String timestampfieldname) {
-        ensureClient();
-        SortBuilder<?> sort = SortBuilders
-                .fieldSort(timestampfieldname)
-                .order(SortOrder.DESC);
-        SearchSourceBuilder builder = new SearchSourceBuilder()
-                .sort(sort)
-                .storedField(timestampfieldname)
-                .size(1);
-        SearchRequest searchRequest = new SearchRequest()
-                .indices(index)
-                .source(builder);
+        ensureClientIsPresent();
+        SortBuilder<?> sort = SortBuilders.fieldSort(timestampfieldname).order(SortOrder.DESC);
+        SearchSourceBuilder builder = new SearchSourceBuilder();
+        builder.sort(sort);
+        builder.storedField(timestampfieldname);
+        builder.size(1);
+        SearchRequest searchRequest = new SearchRequest();
+        searchRequest.indices(index);
+        searchRequest.source(builder);
         SearchResponse searchResponse =
                 client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
         if (searchResponse.getHits().getHits().length == 1) {
@@ -813,7 +461,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     @Override
     public IndexDefinition buildIndexDefinitionFromSettings(String index, Settings settings)
             throws IOException {
-        boolean isEnabled = settings.getAsBoolean("enabled", !(client instanceof MockExtendedClient));
+        boolean isEnabled = settings.getAsBoolean("enabled", false);
         String indexName = settings.get("name", index);
         String fullIndexName;
         String dateTimePattern = settings.get("dateTimePattern");
@@ -833,8 +481,8 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 .setEnabled(isEnabled)
                 .setIndex(indexName)
                 .setFullIndexName(fullIndexName)
-                .setSettingsUrl(settings.get("settings"))
-                .setMappingsUrl(settings.get("mapping"))
+                .setSettings(findSettingsFrom(settings.get("settings")))
+                .setMappings(findMappingsFrom(settings.get("mapping")))
                 .setDateTimePattern(dateTimePattern)
                 .setIgnoreErrors(settings.getAsBoolean("skiperrors", false))
                 .setShift(settings.getAsBoolean("shift", true))
@@ -847,16 +495,10 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     @Override
     public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
-        ensureClient();
+        ensureClientIsPresent();
         if (index == null) {
             throw new IOException("no index name given");
         }
-        if (key == null) {
-            throw new IOException("no key given");
-        }
-        if (value == null) {
-            throw new IOException("no value given");
-        }
         Settings.Builder updateSettingsBuilder = Settings.builder();
         updateSettingsBuilder.put(key, value.toString());
         UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
@@ -864,18 +506,48 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
     }
 
-    private void ensureClient() {
-        if (this instanceof MockExtendedClient) {
-            return;
+    private static String findSettingsFrom(String string) throws IOException {
+        if (string == null) {
+            return null;
         }
-        if (client == null) {
-            throw new IllegalStateException("no client");
+        try {
+            URL url = new URL(string);
+            try (InputStream inputStream = url.openStream()) {
+                Settings settings = Settings.builder().loadFromStream(string, inputStream, true).build();
+                XContentBuilder builder = JsonXContent.contentBuilder();
+                settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
+                return Strings.toString(builder);
+            }
+        } catch (MalformedURLException e) {
+            return string;
         }
     }
 
-    private void ensureIndexGiven(String index) {
-        if (index == null) {
-            throw new IllegalArgumentException("no index given");
+    private static String findMappingsFrom(String string) throws IOException {
+        if (string == null) {
+            return null;
+        }
+        try {
+            URL url = new URL(string);
+            try (InputStream inputStream = url.openStream()) {
+                if (string.endsWith(".json")) {
+                    Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
+                    XContentBuilder builder = JsonXContent.contentBuilder();
+                    builder.startObject().map(mappings).endObject();
+                    return Strings.toString(builder);
+                }
+                if (string.endsWith(".yml") || string.endsWith(".yaml")) {
+                    Map<String, ?> mappings = YamlXContent.yamlXContent.createParser(NamedXContentRegistry.EMPTY,
+                            DeprecationHandler.THROW_UNSUPPORTED_OPERATION, inputStream).mapOrdered();
+                    XContentBuilder builder = JsonXContent.contentBuilder();
+                    builder.startObject().map(mappings).endObject();
+                    return Strings.toString(builder);
+                }
+            }
+            return string;
+        } catch (MalformedInputException e) {
+            return string;
         }
     }
 
@@ -896,7 +568,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     }
 
     public void checkMapping(String index) {
-        ensureClient();
+        ensureClientIsPresent();
         GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(index);
         GetMappingsResponse getMappingsResponse = client.execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
         ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> map = getMappingsResponse.getMappings();
@@ -912,25 +584,25 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     private void checkMapping(String index, String type, MappingMetaData mappingMetaData) {
         try {
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0)
-                    .trackTotalHits(true);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices(index)
-                    .source(builder);
+            SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
+                    .setIndices(index)
+                    .setQuery(QueryBuilders.matchAllQuery())
+                    .setSize(0)
+                    .setTrackTotalHits(true);
             SearchResponse searchResponse =
-                    client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            TotalHits total = searchResponse.getHits().getTotalHits();
-            if (total.value > 0L) {
+                    searchRequestBuilder.execute().actionGet();
+            long total = searchResponse.getHits().getTotalHits().value;
+            if (total > 0L) {
                 Map<String, Long> fields = new TreeMap<>();
                 Map<String, Object> root = mappingMetaData.getSourceAsMap();
-                checkMapping(index, "", "", root, fields);
+                checkMapping(index, type, "", "", root, fields);
                 AtomicInteger empty = new AtomicInteger();
                 Map<String, Long> map = sortByValue(fields);
                 map.forEach((key, value) -> {
                     logger.info("{} {} {}",
-                            key, value, (double) value * 100 / total.value);
+                            key,
+                            value,
+                            (double) value * 100 / total);
                     if (value == 0) {
                         empty.incrementAndGet();
                     }
@@ -944,7 +616,7 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
     }
 
     @SuppressWarnings("unchecked")
-    private void checkMapping(String index,
+    private void checkMapping(String index, String type,
                               String pathDef, String fieldName, Map<String, Object> map,
                               Map<String, Long> fields) {
         String path = pathDef;
@@ -969,20 +641,18 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
                 String fieldType = o instanceof String ? o.toString() : null;
                 // do not recurse into our custom field mapper
                 if (!"standardnumber".equals(fieldType) && !"ref".equals(fieldType)) {
-                    checkMapping(index, path, key, child, fields);
+                    checkMapping(index, type, path, key, child, fields);
                 }
             } else if ("type".equals(key)) {
                 QueryBuilder filterBuilder = QueryBuilders.existsQuery(path);
                 QueryBuilder queryBuilder = QueryBuilders.constantScoreQuery(filterBuilder);
-                SearchSourceBuilder builder = new SearchSourceBuilder()
-                        .query(queryBuilder)
-                        .size(0)
-                        .trackTotalHits(true);
-                SearchRequest searchRequest = new SearchRequest()
-                        .indices(index)
-                        .source(builder);
+                SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
+                        .setIndices(index)
+                        .setQuery(queryBuilder)
+                        .setSize(0)
+                        .setTrackTotalHits(true);
                 SearchResponse searchResponse =
-                        client.execute(SearchAction.INSTANCE, searchRequest).actionGet();
+                        searchRequestBuilder.execute().actionGet();
                 fields.put(path, searchResponse.getHits().getTotalHits().value);
             }
         }
@@ -995,58 +665,38 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         return result;
     }
 
-    private static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
-        switch (timeUnit) {
-            case DAYS:
-                return TimeValue.timeValueHours(24 * timeValue);
-            case HOURS:
-                return TimeValue.timeValueHours(timeValue);
-            case MINUTES:
-                return TimeValue.timeValueMinutes(timeValue);
-            case SECONDS:
-                return TimeValue.timeValueSeconds(timeValue);
-            case MILLISECONDS:
-                return TimeValue.timeValueMillis(timeValue);
-            case MICROSECONDS:
-                return TimeValue.timeValueNanos(1000 * timeValue);
-            case NANOSECONDS:
-                return TimeValue.timeValueNanos(timeValue);
-            default:
-                throw new IllegalArgumentException("unknown time unit: " + timeUnit);
-        }
-    }
-
     private static class SuccessIndexShiftResult implements IndexShiftResult {
 
-        List<String> movedAliases;
+        Collection<String> movedAliases;
 
-        List<String> newAliases;
+        Collection<String> newAliases;
 
-        SuccessIndexShiftResult(List<String> movedAliases, List<String> newAliases) {
+        SuccessIndexShiftResult(Collection<String> movedAliases, Collection<String> newAliases) {
             this.movedAliases = movedAliases;
             this.newAliases = newAliases;
         }
 
         @Override
-        public List<String> getMovedAliases() {
+        public Collection<String> getMovedAliases() {
             return movedAliases;
         }
 
         @Override
-        public List<String> getNewAliases() {
+        public Collection<String> getNewAliases() {
             return newAliases;
         }
     }
 
     private static class SuccessPruneResult implements IndexPruneResult {
 
-        List<String> candidateIndices;
+        Collection<String> candidateIndices;
 
-        List<String> indicesToDelete;
+        Collection<String> indicesToDelete;
 
         AcknowledgedResponse response;
 
-        SuccessPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
+        SuccessPruneResult(Collection<String> candidateIndices,
+                           Collection<String> indicesToDelete,
                            AcknowledgedResponse response) {
             this.candidateIndices = candidateIndices;
             this.indicesToDelete = indicesToDelete;
@@ -1059,48 +709,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         }
 
         @Override
-        public List<String> getCandidateIndices() {
+        public Collection<String> getCandidateIndices() {
             return candidateIndices;
         }
 
         @Override
-        public List<String> getDeletedIndices() {
-            return indicesToDelete;
-        }
-
-        @Override
-        public boolean isAcknowledged() {
-            return response.isAcknowledged();
-        }
-    }
-
-    private static class FailPruneResult implements IndexPruneResult {
-
-        List<String> candidateIndices;
-
-        List<String> indicesToDelete;
-
-        AcknowledgedResponse response;
-
-        FailPruneResult(List<String> candidateIndices, List<String> indicesToDelete,
-                           AcknowledgedResponse response) {
-            this.candidateIndices = candidateIndices;
-            this.indicesToDelete = indicesToDelete;
-            this.response = response;
-        }
-
-        @Override
-        public IndexPruneResult.State getState() {
-            return IndexPruneResult.State.SUCCESS;
-        }
-
-        @Override
-        public List<String> getCandidateIndices() {
-            return candidateIndices;
-        }
-
-        @Override
-        public List<String> getDeletedIndices() {
+        public Collection<String> getDeletedIndices() {
             return indicesToDelete;
         }
 
@@ -1112,11 +726,11 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
 
     private static class NothingToDoPruneResult implements IndexPruneResult {
 
-        List<String> candidateIndices;
+        Collection<String> candidateIndices;
 
-        List<String> indicesToDelete;
+        Collection<String> indicesToDelete;
 
-        NothingToDoPruneResult(List<String> candidateIndices, List<String> indicesToDelete) {
+        NothingToDoPruneResult(Collection<String> candidateIndices, List<String> indicesToDelete) {
             this.candidateIndices = candidateIndices;
             this.indicesToDelete = indicesToDelete;
         }
@@ -1127,12 +741,12 @@ public abstract class AbstractExtendedClient implements ExtendedClient {
         }
 
         @Override
-        public List<String> getCandidateIndices() {
+        public Collection<String> getCandidateIndices() {
             return candidateIndices;
         }
 
         @Override
-        public List<String> getDeletedIndices() {
+        public Collection<String> getDeletedIndices() {
             return indicesToDelete;
         }
 

elx-common/src/main/java/org/xbib/elx/common/AbstractBasicClient.java

@@ -0,0 +1,217 @@
+package org.xbib.elx.common;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.ElasticsearchTimeoutException;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
+import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
+import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction;
+import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.xbib.elx.api.BasicClient;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public abstract class AbstractBasicClient implements BasicClient {
+
+    private static final Logger logger = LogManager.getLogger(AbstractBasicClient.class.getName());
+
+    protected ElasticsearchClient client;
+
+    protected Settings settings;
+
+    private final AtomicBoolean closed;
+
+    public AbstractBasicClient() {
+        closed = new AtomicBoolean(false);
+    }
+
+    @Override
+    public void setClient(ElasticsearchClient client) {
+        logger.log(Level.INFO, "setting client = " + client);
+        this.client = client;
+    }
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return client;
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        if (closed.compareAndSet(false, true)) {
+            logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
+            this.settings = settings;
+            setClient(createClient(settings));
+        } else {
+            logger.log(Level.WARN, "not initializing");
+        }
+    }
+
+    @Override
+    public String getClusterName() {
+        ensureClientIsPresent();
+        try {
+            ClusterStateRequest clusterStateRequest = new ClusterStateRequest().clear();
+            ClusterStateResponse clusterStateResponse =
+                    getClient().execute(ClusterStateAction.INSTANCE, clusterStateRequest).actionGet();
+            return clusterStateResponse.getClusterName().value();
+        } catch (ElasticsearchTimeoutException e) {
+            logger.warn(e.getMessage(), e);
+            return "TIMEOUT";
+        } catch (NoNodeAvailableException e) {
+            logger.warn(e.getMessage(), e);
+            return "DISCONNECTED";
+        } catch (Exception e) {
+            logger.warn(e.getMessage(), e);
+            return "[" + e.getMessage() + "]";
+        }
+    }
+
+    @Override
+    public void waitForCluster(String statusString, long maxWaitTime, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        ClusterHealthStatus status = ClusterHealthStatus.fromString(statusString);
+        TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
+        ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
+                new ClusterHealthRequest().timeout(timeout).waitForStatus(status)).actionGet();
+        if (healthResponse != null && healthResponse.isTimedOut()) {
+            String message = "timeout, cluster state is " + healthResponse.getStatus().name() + " and not " + status.name();
+            if (logger.isErrorEnabled()) {
+                logger.error(message);
+            }
+            throw new IllegalStateException(message);
+        }
+    }
+    @Override
+    public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        logger.info("waiting for cluster shard settling");
+        TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
+        ClusterHealthRequest clusterHealthRequest = new ClusterHealthRequest()
+                .waitForNoInitializingShards(true)
+                .waitForNoRelocatingShards(true)
+                .timeout(timeout);
+        ClusterHealthResponse healthResponse =
+                client.execute(ClusterHealthAction.INSTANCE, clusterHealthRequest).actionGet();
+        if (healthResponse.isTimedOut()) {
+            String message = "timeout waiting for cluster shards";
+            logger.error(message);
+            throw new IllegalStateException(message);
+        }
+    }
+
+    @Override
+    public String getHealthColor(long maxWaitTime, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        try {
+            TimeValue timeout = toTimeValue(maxWaitTime, timeUnit);
+            ClusterHealthResponse healthResponse = client.execute(ClusterHealthAction.INSTANCE,
+                    new ClusterHealthRequest().timeout(timeout)).actionGet();
+            ClusterHealthStatus status = healthResponse.getStatus();
+            return status.name();
+        } catch (ElasticsearchTimeoutException e) {
+            logger.warn(e.getMessage(), e);
+            return "TIMEOUT";
+        } catch (NoNodeAvailableException e) {
+            logger.warn(e.getMessage(), e);
+            return "DISCONNECTED";
+        } catch (Exception e) {
+            logger.warn(e.getMessage(), e);
+            return "[" + e.getMessage() + "]";
+        }
+    }
+
+    @Override
+    public long getSearchableDocs(String index) {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE)
+                .setIndices(index)
+                .setQuery(QueryBuilders.matchAllQuery())
+                .setSize(0)
+                .setTrackTotalHits(true);
+        return searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value;
+    }
+
+    @Override
+    public boolean isIndexExists(String index) {
+        IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
+        indicesExistsRequest.indices(index);
+        IndicesExistsResponse indicesExistsResponse =
+                client.execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
+        return indicesExistsResponse.isExists();
+    }
+
+
+    @Override
+    public void close() throws IOException {
+        ensureClientIsPresent();
+        if (closed.compareAndSet(false, true)) {
+            closeClient(settings);
+        }
+    }
+
+    protected abstract ElasticsearchClient createClient(Settings settings) throws IOException;
+
+    protected abstract void closeClient(Settings settings) throws IOException;
+
+    protected void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
+        ensureClientIsPresent();
+        if (index == null) {
+            throw new IOException("no index name given");
+        }
+        if (key == null) {
+            throw new IOException("no key given");
+        }
+        if (value == null) {
+            throw new IOException("no value given");
+        }
+        Settings.Builder updateSettingsBuilder = Settings.builder();
+        updateSettingsBuilder.put(key, value.toString());
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(index)
+                .settings(updateSettingsBuilder).timeout(toTimeValue(timeout, timeUnit));
+        client.execute(UpdateSettingsAction.INSTANCE, updateSettingsRequest).actionGet();
+    }
+
+    protected void ensureClientIsPresent() {
+        if (client == null) {
+            throw new IllegalStateException("no client");
+        }
+    }
+
+    protected static TimeValue toTimeValue(long timeValue, TimeUnit timeUnit) {
+        switch (timeUnit) {
+            case DAYS:
+                return TimeValue.timeValueHours(24 * timeValue);
+            case HOURS:
+                return TimeValue.timeValueHours(timeValue);
+            case MINUTES:
+                return TimeValue.timeValueMinutes(timeValue);
+            case SECONDS:
+                return TimeValue.timeValueSeconds(timeValue);
+            case MILLISECONDS:
+                return TimeValue.timeValueMillis(timeValue);
+            case MICROSECONDS:
+                return TimeValue.timeValueNanos(1000 * timeValue);
+            case NANOSECONDS:
+                return TimeValue.timeValueNanos(timeValue);
+            default:
+                throw new IllegalArgumentException("unknown time unit: " + timeUnit);
+        }
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/AbstractBulkClient.java

@@ -0,0 +1,246 @@
+package org.xbib.elx.common;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.action.admin.indices.create.CreateIndexAction;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.flush.FlushAction;
+import org.elasticsearch.action.admin.indices.flush.FlushRequest;
+import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
+import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.xbib.elx.api.BulkClient;
+import org.xbib.elx.api.BulkController;
+import org.xbib.elx.api.BulkMetric;
+import org.xbib.elx.api.IndexDefinition;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public abstract class AbstractBulkClient extends AbstractBasicClient implements BulkClient {
+
+    private static final Logger logger = LogManager.getLogger(AbstractBulkClient.class.getName());
+
+    private BulkMetric bulkMetric;
+
+    private BulkController bulkController;
+
+    private final AtomicBoolean closed = new AtomicBoolean(true);
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        if (closed.compareAndSet(true, false)) {
+            super.init(settings);
+            logger.log(Level.INFO, "initializing with settings = " + settings.toDelimitedString(','));
+            bulkMetric = new DefaultBulkMetric();
+            bulkMetric.init(settings);
+            bulkController = new DefaultBulkController(this, bulkMetric);
+            bulkController.init(settings);
+        } else {
+            logger.log(Level.WARN, "not initializing");
+        }
+    }
+
+    @Override
+    public BulkMetric getBulkMetric() {
+        return bulkMetric;
+    }
+
+    @Override
+    public BulkController getBulkController() {
+        return bulkController;
+    }
+
+    @Override
+    public void flush() throws IOException {
+        if (bulkController != null) {
+            bulkController.flush();
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (closed.compareAndSet(false, true)) {
+            ensureClientIsPresent();
+            if (bulkMetric != null) {
+                logger.info("closing bulk metric");
+                bulkMetric.close();
+            }
+            if (bulkController != null) {
+                logger.info("closing bulk controller");
+                bulkController.close();
+            }
+            closeClient(settings);
+        }
+    }
+
+    @Override
+    public void newIndex(IndexDefinition indexDefinition) throws IOException {
+        Settings settings = indexDefinition.getSettings() == null ? null :
+                Settings.builder().loadFromSource(indexDefinition.getSettings(), XContentType.JSON).build();
+        Map<String, ?> mappings = indexDefinition.getMappings() == null ? null :
+                JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexDefinition.getMappings()).mapOrdered();
+        newIndex(indexDefinition.getFullIndexName(), settings, mappings);
+    }
+
+    @Override
+    public void newIndex(String index) throws IOException {
+        newIndex(index, Settings.EMPTY, (Map<String, ?>) null);
+    }
+
+    @Override
+    public void newIndex(String index, Settings settings) throws IOException {
+        newIndex(index, settings, (Map<String, ?>) null);
+    }
+
+    @Override
+    public void newIndex(String index, Settings settings, XContentBuilder builder) throws IOException {
+        String mappingString = Strings.toString(builder);
+        Map<String, ?> mappings = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY,
+                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, mappingString).mapOrdered();
+        newIndex(index, settings, mappings);
+    }
+
+    @Override
+    public void newIndex(String index, Settings settings, Map<String, ?> mapping) throws IOException {
+        if (index == null) {
+            logger.warn("no index name given to create index");
+            return;
+        }
+        ensureClientIsPresent();
+        waitForCluster("YELLOW", 30L, TimeUnit.SECONDS);
+        CreateIndexRequest createIndexRequest = new CreateIndexRequest().index(index);
+        if (settings != null) {
+            createIndexRequest.settings(settings);
+        }
+        if (mapping != null) {
+            createIndexRequest.mapping("_doc", mapping);
+        }
+        CreateIndexResponse createIndexResponse = client.execute(CreateIndexAction.INSTANCE, createIndexRequest).actionGet();
+        if (createIndexResponse.isAcknowledged()) {
+            logger.info("index {} created", index);
+        }
+    }
+
+    @Override
+    public void startBulk(IndexDefinition indexDefinition) throws IOException {
+        startBulk(indexDefinition.getFullIndexName(), -1, 1);
+    }
+
+    @Override
+    public void startBulk(String index, long startRefreshIntervalSeconds, long stopRefreshIntervalSeconds)
+            throws IOException {
+        if (bulkController != null) {
+            ensureClientIsPresent();
+            bulkController.startBulkMode(index, startRefreshIntervalSeconds, stopRefreshIntervalSeconds);
+        }
+    }
+
+    @Override
+    public void stopBulk(IndexDefinition indexDefinition) throws IOException {
+        if (bulkController != null) {
+            ensureClientIsPresent();
+            bulkController.stopBulkMode(indexDefinition);
+        }
+    }
+
+    @Override
+    public void stopBulk(String index, long timeout, TimeUnit timeUnit) throws IOException {
+        if (bulkController != null) {
+            ensureClientIsPresent();
+            bulkController.stopBulkMode(index, timeout, timeUnit);
+        }
+    }
+
+    @Override
+    public BulkClient index(String index, String id, boolean create, String source) {
+        return index(index, id, create, new BytesArray(source.getBytes(StandardCharsets.UTF_8)));
+    }
+
+    @Override
+    public BulkClient index(String index, String id, boolean create, BytesReference source) {
+        return index(new IndexRequest().index(index).id(id).create(create)
+                .source(source, XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient index(IndexRequest indexRequest) {
+        ensureClientIsPresent();
+        bulkController.bulkIndex(indexRequest);
+        return this;
+    }
+
+    @Override
+    public BulkClient delete(String index, String id) {
+        return delete(new DeleteRequest().index(index).id(id));
+    }
+
+    @Override
+    public BulkClient delete(DeleteRequest deleteRequest) {
+        ensureClientIsPresent();
+        bulkController.bulkDelete(deleteRequest);
+        return this;
+    }
+
+    @Override
+    public BulkClient update(String index, String id, BytesReference source) {
+        return update(new UpdateRequest().index(index).id(id)
+                .doc(source, XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient update(String index, String id, String source) {
+        return update(new UpdateRequest().index(index).id(id)
+                .doc(source.getBytes(StandardCharsets.UTF_8), XContentType.JSON));
+    }
+
+    @Override
+    public BulkClient update(UpdateRequest updateRequest) {
+        ensureClientIsPresent();
+        bulkController.bulkUpdate(updateRequest);
+        return this;
+    }
+
+    @Override
+    public boolean waitForResponses(long timeout, TimeUnit timeUnit) {
+        ensureClientIsPresent();
+        return bulkController.waitForBulkResponses(timeout, timeUnit);
+    }
+
+    @Override
+    public void updateIndexSetting(String index, String key, Object value, long timeout, TimeUnit timeUnit) throws IOException {
+        super.updateIndexSetting(index, key, value, timeout, timeUnit);
+    }
+
+    @Override
+    public void flushIndex(String index) {
+        if (index != null) {
+            ensureClientIsPresent();
+            client.execute(FlushAction.INSTANCE, new FlushRequest(index)).actionGet();
+        }
+    }
+
+    @Override
+    public void refreshIndex(String index) {
+        if (index != null) {
+            ensureClientIsPresent();
+            client.execute(RefreshAction.INSTANCE, new RefreshRequest(index)).actionGet();
+        }
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/AbstractSearchClient.java

@@ -0,0 +1,158 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.get.GetAction;
+import org.elasticsearch.action.get.GetRequestBuilder;
+import org.elasticsearch.action.get.GetResponse;
+import org.elasticsearch.action.get.MultiGetAction;
+import org.elasticsearch.action.get.MultiGetRequestBuilder;
+import org.elasticsearch.action.get.MultiGetResponse;
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequestBuilder;
+import org.elasticsearch.action.search.SearchAction;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.search.SearchScrollRequestBuilder;
+import org.elasticsearch.action.search.ShardSearchFailure;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.search.SearchHit;
+import org.xbib.elx.api.SearchClient;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Optional;
+import java.util.Spliterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+public abstract class AbstractSearchClient extends AbstractBasicClient implements SearchClient {
+
+    @Override
+    public Optional<GetResponse> get(Consumer<GetRequestBuilder> getRequestBuilderConsumer) {
+        GetRequestBuilder getRequestBuilder = new GetRequestBuilder(client, GetAction.INSTANCE);
+        getRequestBuilderConsumer.accept(getRequestBuilder);
+        GetResponse getResponse = getRequestBuilder.execute().actionGet();
+        return getResponse.isExists() ? Optional.of(getResponse) : Optional.empty();
+    }
+
+    @Override
+    public Optional<MultiGetResponse> multiGet(Consumer<MultiGetRequestBuilder> multiGetRequestBuilderConsumer) {
+        MultiGetRequestBuilder multiGetRequestBuilder = new MultiGetRequestBuilder(client, MultiGetAction.INSTANCE);
+        multiGetRequestBuilderConsumer.accept(multiGetRequestBuilder);
+        MultiGetResponse multiGetItemResponse = multiGetRequestBuilder.execute().actionGet();
+        return multiGetItemResponse.getResponses().length == 0 ?  Optional.empty() : Optional.of(multiGetItemResponse);
+    }
+
+    @Override
+    public Optional<SearchResponse> search(Consumer<SearchRequestBuilder> queryBuilder) {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
+        queryBuilder.accept(searchRequestBuilder);
+        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
+        if (searchResponse.getFailedShards() > 0) {
+            StringBuilder sb = new StringBuilder("Search failed:");
+            for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
+                sb.append("\n").append(failure.reason());
+            }
+            throw new ElasticsearchException(sb.toString());
+        }
+        return searchResponse.getHits().getHits().length == 0 ? Optional.empty() : Optional.of(searchResponse);
+    }
+
+    @Override
+    public Stream<SearchHit> search(Consumer<SearchRequestBuilder> queryBuilder,
+                                         TimeValue scrollTime, int scrollSize) {
+        SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
+        queryBuilder.accept(searchRequestBuilder);
+        searchRequestBuilder.setScroll(scrollTime).setSize(scrollSize);
+        SearchResponse originalSearchResponse = searchRequestBuilder.execute().actionGet();
+        Stream<SearchResponse> infiniteResponses = Stream.iterate(originalSearchResponse,
+                searchResponse -> new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE)
+                        .setScrollId(searchResponse.getScrollId())
+                        .setScroll(scrollTime)
+                        .execute().actionGet());
+        Predicate<SearchResponse> condition = searchResponse -> searchResponse.getHits().getHits().length > 0;
+        Consumer<SearchResponse> lastAction = searchResponse -> {
+            ClearScrollRequestBuilder clearScrollRequestBuilder =
+                    new ClearScrollRequestBuilder(client, ClearScrollAction.INSTANCE)
+                            .addScrollId(searchResponse.getScrollId());
+            clearScrollRequestBuilder.execute().actionGet();
+        };
+        return StreamSupport.stream(TakeWhileSpliterator.over(infiniteResponses.spliterator(),
+                condition, lastAction), false)
+                .onClose(infiniteResponses::close)
+                .flatMap(searchResponse -> Arrays.stream(searchResponse.getHits().getHits()));
+    }
+
+    @Override
+    public Stream<String> getIds(Consumer<SearchRequestBuilder> queryBuilder) {
+        return search(queryBuilder, TimeValue.timeValueMinutes(1), 1000).map(SearchHit::getId);
+    }
+
+    static class TakeWhileSpliterator<T> implements Spliterator<T> {
+
+        private final Spliterator<T> source;
+
+        private final Predicate<T> condition;
+
+        private final Consumer<T> lastElement;
+
+        private final boolean inclusive;
+
+        private final AtomicBoolean checked;
+
+        static <T> TakeWhileSpliterator<T> over(Spliterator<T> source,
+                                                Predicate<T> condition,
+                                                Consumer<T> lastElement) {
+            return new TakeWhileSpliterator<>(source, condition, lastElement, true);
+        }
+
+        private TakeWhileSpliterator(Spliterator<T> source,
+                                     Predicate<T> condition,
+                                     Consumer<T> lastElement,
+                                     boolean inclusive) {
+            this.source = source;
+            this.condition = condition;
+            this.lastElement = lastElement;
+            this.inclusive = inclusive;
+            this.checked = new AtomicBoolean(true);
+        }
+
+        @Override
+        public boolean tryAdvance(Consumer<? super T> action) {
+            return checked.get() && source.tryAdvance(e -> {
+                if (condition.test(e)) {
+                    action.accept(e);
+                } else {
+                    if (inclusive && checked.get()) {
+                        action.accept(e);
+                    }
+                    lastElement.accept(e);
+                    checked.set(false);
+                }
+            });
+        }
+
+        @Override
+        public Spliterator<T> trySplit() {
+            return null;
+        }
+
+        @Override
+        public long estimateSize() {
+            return checked.get() ? source.estimateSize() : 0;
+        }
+
+        @Override
+        public int characteristics() {
+            return source.characteristics() &~ Spliterator.SIZED;
+        }
+
+        @Override
+        public Comparator<? super T> getComparator() {
+            return source.getComparator();
+        }
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/ClientBuilder.java

@@ -8,11 +8,12 @@ import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
-import org.xbib.elx.api.ExtendedClient;
-import org.xbib.elx.api.ExtendedClientProvider;
+import org.xbib.elx.api.AdminClientProvider;
+import org.xbib.elx.api.BulkClientProvider;
+import org.xbib.elx.api.BasicClient;
+import org.xbib.elx.api.SearchClientProvider;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.ServiceLoader;
 
@@ -23,11 +24,15 @@ public class ClientBuilder {
 
     private final ElasticsearchClient client;
 
+    private final ClassLoader classLoader;
+
     private final Settings.Builder settingsBuilder;
 
-    private Map<Class<? extends ExtendedClientProvider>, ExtendedClientProvider> providerMap;
+    private Class<? extends AdminClientProvider> adminClientProvider;
 
-    private Class<? extends ExtendedClientProvider> provider;
+    private Class<? extends BulkClientProvider> bulkClientProvider;
+
+    private Class<? extends SearchClientProvider> searchClientProvider;
 
     public ClientBuilder() {
         this(null);
@@ -39,14 +44,9 @@ public class ClientBuilder {
 
     public ClientBuilder(ElasticsearchClient client, ClassLoader classLoader) {
         this.client = client;
+        this.classLoader = classLoader;
         this.settingsBuilder = Settings.builder();
         settingsBuilder.put("node.name", "elx-client-" + Version.CURRENT);
-        this.providerMap = new HashMap<>();
-        ServiceLoader<ExtendedClientProvider> serviceLoader = ServiceLoader.load(ExtendedClientProvider.class,
-                classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader());
-        for (ExtendedClientProvider provider : serviceLoader) {
-            providerMap.put(provider.getClass(), provider);
-        }
     }
 
     public static ClientBuilder builder() {
@@ -57,8 +57,18 @@ public class ClientBuilder {
         return new ClientBuilder(client);
     }
 
-    public ClientBuilder provider(Class<? extends ExtendedClientProvider> provider) {
-        this.provider = provider;
+    public ClientBuilder setAdminClientProvider(Class<? extends AdminClientProvider> adminClientProvider) {
+        this.adminClientProvider = adminClientProvider;
+        return this;
+    }
+
+    public ClientBuilder setBulkClientProvider(Class<? extends BulkClientProvider> bulkClientProvider) {
+        this.bulkClientProvider = bulkClientProvider;
+        return this;
+    }
+
+    public ClientBuilder setSearchClientProvider(Class<? extends SearchClientProvider> searchClientProvider) {
+        this.searchClientProvider = searchClientProvider;
         return this;
     }
 
@@ -97,15 +107,59 @@ public class ClientBuilder {
         return this;
     }
 
-    @SuppressWarnings("unchecked")
-    public <C extends ExtendedClient> C build() throws IOException {
-        if (provider == null) {
-            throw new IllegalArgumentException("no provider");
+    public ClientBuilder put(Map<String, ?> map) {
+        for (Map.Entry<String, ?> entry : map.entrySet()) {
+            if (entry.getValue() == null) {
+                continue;
             }
+            if (entry.getValue() instanceof String ||
+                    entry.getValue() instanceof  Integer ||
+                    entry.getValue() instanceof Long ||
+                    entry.getValue() instanceof Float ||
+                    entry.getValue() instanceof TimeValue) {
+                settingsBuilder.put(entry.getKey(), entry.getValue().toString());
+            } else {
+                logger.log(Level.WARN, "skipping " + entry.getValue() +
+                        " because invalid class type " + entry.getValue().getClass().getName());
+            }
+        }
+        return this;
+    }
+
+    @SuppressWarnings("unchecked")
+    public <C extends BasicClient> C build() throws IOException {
         Settings settings = settingsBuilder.build();
         logger.log(Level.INFO, "settings = " + settings.toDelimitedString(','));
-        return (C) providerMap.get(provider).getExtendedClient()
-                .setClient(client)
-                .init(settings);
+        if (adminClientProvider != null) {
+            for (AdminClientProvider provider : ServiceLoader.load(AdminClientProvider.class, classLoader)) {
+                if (provider.getClass().isAssignableFrom(adminClientProvider)) {
+                    C c = (C) provider.getClient();
+                    c.setClient(client);
+                    c.init(settings);
+                    return c;
+                }
+            }
+        }
+        if (bulkClientProvider != null) {
+            for (BulkClientProvider provider : ServiceLoader.load(BulkClientProvider.class, classLoader)) {
+                if (provider.getClass().isAssignableFrom(bulkClientProvider)) {
+                    C c = (C) provider.getClient();
+                    c.setClient(client);
+                    c.init(settings);
+                    return c;
+                }
+            }
+        }
+        if (searchClientProvider != null) {
+            for (SearchClientProvider provider : ServiceLoader.load(SearchClientProvider.class, classLoader)) {
+                if (provider.getClass().isAssignableFrom(searchClientProvider)) {
+                    C c = (C) provider.getClient();
+                    c.setClient(client);
+                    c.init(settings);
+                    return c;
+                }
+            }
+        }
+        throw new IllegalArgumentException("no provider");
     }
 }

elx-common/src/main/java/org/xbib/elx/common/DefaultBulkController.java

@@ -8,11 +8,11 @@ import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
+import org.xbib.elx.api.BulkClient;
 import org.xbib.elx.api.BulkController;
 import org.xbib.elx.api.BulkListener;
 import org.xbib.elx.api.BulkMetric;
 import org.xbib.elx.api.BulkProcessor;
-import org.xbib.elx.api.ExtendedClient;
 import org.xbib.elx.api.IndexDefinition;
 
 import java.io.IOException;
@@ -27,7 +27,7 @@ public class DefaultBulkController implements BulkController {
 
     private static final Logger logger = LogManager.getLogger(DefaultBulkController.class);
 
-    private final ExtendedClient client;
+    private final BulkClient bulkClient;
 
     private final BulkMetric bulkMetric;
 
@@ -45,9 +45,9 @@ public class DefaultBulkController implements BulkController {
 
     private final AtomicBoolean active;
 
-    public DefaultBulkController(ExtendedClient client) {
-        this.client = client;
-        this.bulkMetric = new DefaultBulkMetric();
+    public DefaultBulkController(BulkClient bulkClient, BulkMetric bulkMetric) {
+        this.bulkClient = bulkClient;
+        this.bulkMetric = bulkMetric;
         this.indexNames = new ArrayList<>();
         this.active = new AtomicBoolean(false);
         this.startBulkRefreshIntervals = new HashMap<>();
@@ -80,7 +80,7 @@ public class DefaultBulkController implements BulkController {
         boolean enableBulkLogging = settings.getAsBoolean(Parameters.ENABLE_BULK_LOGGING.name(),
                 Parameters.ENABLE_BULK_LOGGING.getValue());
         BulkListener bulkListener = new DefaultBulkListener(this, bulkMetric, enableBulkLogging);
-        this.bulkProcessor = DefaultBulkProcessor.builder(client.getClient(), bulkListener)
+        this.bulkProcessor = DefaultBulkProcessor.builder(bulkClient.getClient(), bulkListener)
                 .setBulkActions(maxActionsPerRequest)
                 .setConcurrentRequests(maxConcurrentRequests)
                 .setFlushInterval(flushIngestInterval)
@@ -115,7 +115,7 @@ public class DefaultBulkController implements BulkController {
             startBulkRefreshIntervals.put(indexName, startRefreshIntervalInSeconds);
             stopBulkRefreshIntervals.put(indexName, stopRefreshIntervalInSeconds);
             if (startRefreshIntervalInSeconds != 0L) {
-                client.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s",
+                bulkClient.updateIndexSetting(indexName, "refresh_interval", startRefreshIntervalInSeconds + "s",
                         30L, TimeUnit.SECONDS);
             }
         }
@@ -187,7 +187,7 @@ public class DefaultBulkController implements BulkController {
             if (indexNames.contains(index)) {
                 Long secs = stopBulkRefreshIntervals.get(index);
                 if (secs != null && secs != 0L) {
-                    client.updateIndexSetting(index, "refresh_interval", secs + "s",
+                    bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s",
                             30L, TimeUnit.SECONDS);
                 }
                 indexNames.remove(index);
@@ -206,11 +206,11 @@ public class DefaultBulkController implements BulkController {
     public void close() throws IOException {
         flush();
         bulkMetric.close();
-        if (client.waitForResponses(maxWaitTime, maxWaitTimeUnit)) {
+        if (bulkClient.waitForResponses(maxWaitTime, maxWaitTimeUnit)) {
             for (String index : indexNames) {
                 Long secs = stopBulkRefreshIntervals.get(index);
                 if (secs != null && secs != 0L)
-                client.updateIndexSetting(index, "refresh_interval", secs + "s",
+                bulkClient.updateIndexSetting(index, "refresh_interval", secs + "s",
                         30L, TimeUnit.SECONDS);
             }
             indexNames.clear();

elx-common/src/main/java/org/xbib/elx/common/DefaultBulkMetric.java

@@ -1,5 +1,6 @@
 package org.xbib.elx.common;
 
+import org.elasticsearch.common.settings.Settings;
 import org.xbib.elx.api.BulkMetric;
 import org.xbib.metrics.api.Count;
 import org.xbib.metrics.api.Metered;
@@ -36,6 +37,10 @@ public class DefaultBulkMetric implements BulkMetric {
         submitted = new CountMetric();
         succeeded = new CountMetric();
         failed = new CountMetric();
+    }
+
+    @Override
+    public void init(Settings settings) {
         start();
     }
 

elx-common/src/main/java/org/xbib/elx/common/DefaultIndexDefinition.java

@@ -3,8 +3,6 @@ package org.xbib.elx.common;
 import org.xbib.elx.api.IndexDefinition;
 import org.xbib.elx.api.IndexRetention;
 
-import java.net.MalformedURLException;
-import java.net.URL;
 import java.util.concurrent.TimeUnit;
 
 public class DefaultIndexDefinition implements IndexDefinition {
@@ -15,9 +13,9 @@ public class DefaultIndexDefinition implements IndexDefinition {
 
     private String dateTimePattern;
 
-    private URL settingsUrl;
+    private String settings;
 
-    private URL mappingsUrl;
+    private String mappings;
 
     private boolean enabled;
 
@@ -62,37 +60,25 @@ public class DefaultIndexDefinition implements IndexDefinition {
     }
 
     @Override
-    public IndexDefinition setSettingsUrl(String settingsUrlString) throws MalformedURLException {
-        this.settingsUrl = settingsUrlString != null ? new URL(settingsUrlString) : null;
+    public IndexDefinition setSettings(String settings) {
+        this.settings = settings;
         return this;
     }
 
     @Override
-    public IndexDefinition setSettingsUrl(URL settingsUrl) {
-        this.settingsUrl = settingsUrl;
+    public String getSettings() {
+        return settings;
+    }
+
+    @Override
+    public IndexDefinition setMappings(String mappings) {
+        this.mappings = mappings;
         return this;
     }
 
     @Override
-    public URL getSettingsUrl() {
-        return settingsUrl;
-    }
-
-    @Override
-    public IndexDefinition setMappingsUrl(String mappingsUrlString) throws MalformedURLException {
-        this.mappingsUrl = mappingsUrlString != null ? new URL(mappingsUrlString) : null;
-        return this;
-    }
-
-    @Override
-    public IndexDefinition setMappingsUrl(URL mappingsUrl) {
-        this.mappingsUrl = mappingsUrl;
-        return this;
-    }
-
-    @Override
-    public URL getMappingsUrl() {
-        return mappingsUrl;
+    public String getMappings() {
+        return mappings;
     }
 
     @Override

elx-common/src/main/java/org/xbib/elx/common/MockAdminClient.java

@@ -0,0 +1,59 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A mocked client, it does not perform any actions on a cluster. Useful for testing.
+ */
+public class MockAdminClient extends AbstractAdminClient {
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return null;
+    }
+
+    @Override
+    public void init(Settings settings) {
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) {
+        return null;
+    }
+
+    @Override
+    protected void closeClient(Settings settings) {
+    }
+
+    @Override
+    public MockAdminClient deleteIndex(String index) {
+        return this;
+    }
+
+    @Override
+    public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
+        return true;
+    }
+
+    @Override
+    public void waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
+    }
+
+    @Override
+    public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
+
+    }
+
+    @Override
+    public MockAdminClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
+        return this;
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockAdminClientProvider.java

@@ -0,0 +1,10 @@
+package org.xbib.elx.common;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class MockAdminClientProvider implements AdminClientProvider<MockAdminClient> {
+    @Override
+    public MockAdminClient getClient() {
+        return new MockAdminClient();
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockBulkClient.java

@@ -0,0 +1,103 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.update.UpdateRequest;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A mocked client, it does not perform any actions on a cluster. Useful for testing.
+ */
+public class MockBulkClient extends AbstractBulkClient {
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return null;
+    }
+
+    @Override
+    public void init(Settings settings) {
+    }
+
+    @Override
+    public String getClusterName() {
+        return null;
+    }
+
+    @Override
+    public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) {
+        return null;
+    }
+
+    @Override
+    protected void closeClient(Settings settings) {
+    }
+
+    @Override
+    public MockBulkClient index(String index, String id, boolean create, String source) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient delete(String index, String id) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient update(String index, String id, String source) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient index(IndexRequest indexRequest) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient delete(DeleteRequest deleteRequest) {
+        return this;
+    }
+
+    @Override
+    public MockBulkClient update(UpdateRequest updateRequest) {
+        return this;
+    }
+
+    @Override
+    public void startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
+    }
+
+    @Override
+    public void stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
+    }
+
+    @Override
+    public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
+        return true;
+    }
+
+    @Override
+    public void refreshIndex(String index) {
+    }
+
+    @Override
+    public void flushIndex(String index) {
+    }
+
+
+    @Override
+    public void flush() {
+        // nothing to do
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.common;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class MockBulkClientProvider implements BulkClientProvider<MockBulkClient> {
+
+    @Override
+    public MockBulkClient getClient() {
+        return new MockBulkClient();
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockExtendedClient.java

@@ -1,129 +0,0 @@
-package org.xbib.elx.common;
-
-import org.elasticsearch.action.delete.DeleteRequest;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.action.update.UpdateRequest;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.settings.Settings;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * A mocked client, it does not perform any actions on a cluster. Useful for testing.
- */
-public class MockExtendedClient extends AbstractExtendedClient {
-
-    @Override
-    public ElasticsearchClient getClient() {
-        return null;
-    }
-
-    @Override
-    public MockExtendedClient init(Settings settings) {
-        return this;
-    }
-
-    @Override
-    protected ElasticsearchClient createClient(Settings settings) {
-        return null;
-    }
-
-    @Override
-    protected void closeClient() {
-    }
-
-    @Override
-    public MockExtendedClient index(String index, String id, boolean create, String source) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient delete(String index, String id) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient update(String index, String id, String source) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient index(IndexRequest indexRequest) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient delete(DeleteRequest deleteRequest) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient update(UpdateRequest updateRequest) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient startBulk(String index, long startRefreshInterval, long stopRefreshIterval) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient stopBulk(String index, long maxWaitTime, TimeUnit timeUnit) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient newIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient deleteIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient refreshIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public MockExtendedClient flushIndex(String index) {
-        return this;
-    }
-
-    @Override
-    public boolean forceMerge(String index, long maxWaitTime, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public boolean waitForCluster(String healthColor, long timeValue, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public boolean waitForResponses(long maxWaitTime, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public boolean waitForRecovery(String index, long maxWaitTime, TimeUnit timeUnit) {
-        return true;
-    }
-
-    @Override
-    public MockExtendedClient updateReplicaLevel(String index, int level, long maxWaitTime, TimeUnit timeUnit) {
-        return this;
-    }
-
-    @Override
-    public void flush() {
-        // nothing to do
-    }
-
-    @Override
-    public void close() {
-        // nothing to do
-    }
-}

elx-common/src/main/java/org/xbib/elx/common/MockExtendedClientProvider.java

@@ -1,10 +0,0 @@
-package org.xbib.elx.common;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class MockExtendedClientProvider implements ExtendedClientProvider<MockExtendedClient> {
-    @Override
-    public MockExtendedClient getExtendedClient() {
-        return new MockExtendedClient();
-    }
-}

elx-common/src/main/java/org/xbib/elx/common/MockSearchClient.java

@@ -0,0 +1,44 @@
+package org.xbib.elx.common;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A mocked client, it does not perform any actions on a cluster. Useful for testing.
+ */
+public class MockSearchClient extends AbstractSearchClient {
+
+    @Override
+    public ElasticsearchClient getClient() {
+        return null;
+    }
+
+    @Override
+    public void init(Settings settings) {
+    }
+
+    @Override
+    public String getClusterName() {
+        return null;
+    }
+
+    @Override
+    public void waitForShards(long maxWaitTime, TimeUnit timeUnit) {
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) {
+        return null;
+    }
+
+    @Override
+    protected void closeClient(Settings settings) {
+    }
+
+    @Override
+    public void close() {
+        // nothing to do
+    }
+}

elx-common/src/main/java/org/xbib/elx/common/MockSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.common;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class MockSearchClientProvider implements SearchClientProvider<MockSearchClient> {
+
+    @Override
+    public MockSearchClient getClient() {
+        return new MockSearchClient();
+    }
+}

elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.common.MockAdminClientProvider

elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.common.MockBulkClientProvider

elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.common.MockExtendedClientProvider

elx-common/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.common.MockSearchClientProvider

elx-common/src/test/java/org/xbib/elx/common/test/MockClientProviderTest.java

@@ -0,0 +1,41 @@
+package org.xbib.elx.common.test;
+
+import org.junit.jupiter.api.Test;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.MockAdminClient;
+import org.xbib.elx.common.MockAdminClientProvider;
+import org.xbib.elx.common.MockBulkClient;
+import org.xbib.elx.common.MockBulkClientProvider;
+import org.xbib.elx.common.MockSearchClient;
+import org.xbib.elx.common.MockSearchClientProvider;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+class MockClientProviderTest {
+
+    @Test
+    void testMockAdminProvider() throws IOException {
+        MockAdminClient client = ClientBuilder.builder()
+                .setAdminClientProvider(MockAdminClientProvider.class)
+                .build();
+        assertNotNull(client);
+    }
+
+    @Test
+    void testMockBulkProvider() throws IOException {
+        MockBulkClient client = ClientBuilder.builder()
+                .setBulkClientProvider(MockBulkClientProvider.class)
+                .build();
+        assertNotNull(client);
+    }
+
+    @Test
+    void testMockSearchProvider() throws IOException {
+        MockSearchClient client = ClientBuilder.builder()
+                .setSearchClientProvider(MockSearchClientProvider.class)
+                .build();
+        assertNotNull(client);
+    }
+}

elx-common/src/test/java/org/xbib/elx/common/test/MockExtendedClientProviderTest.java

@@ -1,19 +0,0 @@
-package org.xbib.elx.common.test;
-
-import org.junit.jupiter.api.Test;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.common.MockExtendedClient;
-import org.xbib.elx.common.MockExtendedClientProvider;
-
-import java.io.IOException;
-
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-
-class MockExtendedClientProviderTest {
-
-    @Test
-    void testMockExtendedProvider() throws IOException {
-        MockExtendedClient client = ClientBuilder.builder().provider(MockExtendedClientProvider.class).build();
-        assertNotNull(client);
-    }
-}

elx-http/src/main/java/org/xbib/elx/http/ExtendedHttpClientProvider.java

@@ -1,12 +0,0 @@
-package org.xbib.elx.http;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-import java.util.Collections;
-
-public class ExtendedHttpClientProvider implements ExtendedClientProvider<ExtendedHttpClient> {
-    @Override
-    public ExtendedHttpClient getExtendedClient() {
-        return new ExtendedHttpClient(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
-    }
-}

elx-http/src/main/java/org/xbib/elx/http/HttpActionContext.java

@@ -13,7 +13,7 @@ import org.xbib.netty.http.common.HttpResponse;
  */
 public class HttpActionContext<R extends ActionRequest, T extends ActionResponse> {
 
-    private final ExtendedHttpClient extendedHttpClient;
+    private final HttpClientHelper extendedHttpClient;
 
     private final R request;
 
@@ -23,13 +23,13 @@ public class HttpActionContext<R extends ActionRequest, T extends ActionResponse
 
     private HttpResponse httpResponse;
 
-    HttpActionContext(ExtendedHttpClient extendedHttpClient, R request, String url) {
+    HttpActionContext(HttpClientHelper extendedHttpClient, R request, String url) {
         this.extendedHttpClient = extendedHttpClient;
         this.request = request;
         this.url = url;
     }
 
-    public ExtendedHttpClient getExtendedHttpClient() {
+    public HttpClientHelper getExtendedHttpClient() {
         return extendedHttpClient;
     }
 

elx-http/src/main/java/org/xbib/elx/http/HttpAdminClient.java

@@ -0,0 +1,55 @@
+package org.xbib.elx.http;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elx.common.AbstractAdminClient;
+import java.io.IOException;
+
+/**
+ * Elasticsearch HTTP admin client.
+ */
+public class HttpAdminClient extends AbstractAdminClient implements ElasticsearchClient {
+
+    private final HttpClientHelper helper;
+
+    public HttpAdminClient() {
+        this.helper = new HttpClientHelper();
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init(settings);
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return this;
+    }
+
+    @Override
+    protected void closeClient(Settings settings) throws IOException {
+        helper.closeClient(settings);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
+        return helper.execute(action, request);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+        helper.execute(action, request, listener);
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return helper.threadPool();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpAdminClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.http;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class HttpAdminClientProvider implements AdminClientProvider<HttpAdminClient> {
+
+    @Override
+    public HttpAdminClient getClient() {
+        return new HttpAdminClient();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpBulkClient.java

@@ -0,0 +1,55 @@
+package org.xbib.elx.http;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elx.common.AbstractBulkClient;
+import java.io.IOException;
+
+/**
+ * Elasticsearch HTTP bulk client.
+ */
+public class HttpBulkClient extends AbstractBulkClient implements ElasticsearchClient {
+
+    private final HttpClientHelper helper;
+
+    public HttpBulkClient() {
+        this.helper = new HttpClientHelper();
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init(settings);
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return this;
+    }
+
+    @Override
+    protected void closeClient(Settings settings) throws IOException {
+        helper.closeClient(settings);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
+        return helper.execute(action, request);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+        helper.execute(action, request, listener);
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return helper.threadPool();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.http;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class HttpBulkClientProvider implements BulkClientProvider<HttpBulkClient> {
+
+    @Override
+    public HttpBulkClient getClient() {
+        return new HttpBulkClient();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpClientHelper.java

@@ -10,16 +10,15 @@ import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.support.PlainActionFuture;
-import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.xbib.elx.common.AbstractExtendedClient;
 import org.xbib.net.URL;
 import org.xbib.netty.http.client.Client;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,9 +30,9 @@ import java.util.stream.Stream;
 /**
  * Elasticsearch HTTP client.
  */
-public class ExtendedHttpClient extends AbstractExtendedClient implements ElasticsearchClient {
+public class HttpClientHelper {
 
-    private static final Logger logger = LogManager.getLogger(ExtendedHttpClient.class);
+    private static final Logger logger = LogManager.getLogger(HttpClientHelper.class);
 
     private Client nettyHttpClient;
 
@@ -46,17 +45,20 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
 
     private String url;
 
-    public ExtendedHttpClient(List<NamedXContentRegistry.Entry> namedXContentEntries, ClassLoader classLoader) {
+    public HttpClientHelper() {
+        this(Collections.emptyList(), Thread.currentThread().getContextClassLoader());
+    }
+
+    public HttpClientHelper(List<NamedXContentRegistry.Entry> namedXContentEntries,
+                            ClassLoader classLoader) {
         this.registry = new NamedXContentRegistry(Stream.of(getNamedXContents().stream(),
                 namedXContentEntries.stream()).flatMap(Function.identity()).collect(Collectors.toList()));
         this.classLoader = classLoader != null ? classLoader : Thread.currentThread().getContextClassLoader();
         this.actionMap = new HashMap<>();
     }
 
-    @Override
     @SuppressWarnings({"unchecked", "rawtypes"})
-    public ExtendedHttpClient init(Settings settings) throws IOException {
-        super.init(settings);
+    public HttpClientHelper init(Settings settings) throws IOException {
         if (settings.hasValue("url")) {
             this.url = settings.get("url");
         } else if (settings.hasValue("host")) {
@@ -92,22 +94,10 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
         return nettyHttpClient;
     }
 
-    @Override
-    public ElasticsearchClient getClient() {
-        return this;
-    }
-
-    @Override
-    protected ElasticsearchClient createClient(Settings settings) {
-        return this;
-    }
-
-    @Override
-    protected void closeClient() throws IOException {
+    protected void closeClient(Settings settings) throws IOException {
         nettyHttpClient.shutdownGracefully();
     }
 
-    @Override
     public <Request extends ActionRequest, Response extends ActionResponse>
     ActionFuture<Response> execute(ActionType<Response> action, Request request) {
         PlainActionFuture<Response> actionFuture = PlainActionFuture.newFuture();
@@ -115,13 +105,11 @@ public class ExtendedHttpClient extends AbstractExtendedClient implements Elasti
         return actionFuture;
     }
 
-    @Override
     public <Request extends ActionRequest, Response extends ActionResponse>
     void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
         doExecute(action, request, listener);
     }
 
-    @Override
     public ThreadPool threadPool() {
         logger.log(Level.DEBUG, "returning null for threadPool() request");
         return null;

elx-http/src/main/java/org/xbib/elx/http/HttpSearchClient.java

@@ -0,0 +1,55 @@
+package org.xbib.elx.http;
+
+import org.elasticsearch.action.ActionFuture;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.ActionType;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.xbib.elx.common.AbstractSearchClient;
+import java.io.IOException;
+
+/**
+ * Elasticsearch HTTP search client.
+ */
+public class HttpSearchClient extends AbstractSearchClient implements ElasticsearchClient {
+
+    private final HttpClientHelper helper;
+
+    public HttpSearchClient() {
+        this.helper = new HttpClientHelper();
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init(settings);
+    }
+
+    @Override
+    protected ElasticsearchClient createClient(Settings settings) throws IOException {
+        return this;
+    }
+
+    @Override
+    protected void closeClient(Settings settings) throws IOException {
+        helper.closeClient(settings);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> ActionFuture<Response> execute(ActionType<Response> action, Request request) {
+        return helper.execute(action, request);
+    }
+
+    @Override
+    public <Request extends ActionRequest, Response extends ActionResponse> void execute(ActionType<Response> action, Request request, ActionListener<Response> listener) {
+        helper.execute(action, request, listener);
+    }
+
+    @Override
+    public ThreadPool threadPool() {
+        return helper.threadPool();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/HttpSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.http;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class HttpSearchClientProvider implements SearchClientProvider<HttpSearchClient> {
+
+    @Override
+    public HttpSearchClient getClient() {
+        return new HttpSearchClient();
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/action/admin/indices/get/HtppGetIndexAction.java

@@ -0,0 +1,35 @@
+package org.xbib.elx.http.action.admin.indices.get;
+
+import org.elasticsearch.action.admin.indices.get.GetIndexAction;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.xbib.elx.http.HttpAction;
+import org.xbib.netty.http.client.api.Request;
+import org.xbib.netty.http.common.HttpResponse;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+public class HtppGetIndexAction extends HttpAction<GetIndexRequest, GetIndexResponse> {
+
+    @Override
+    public GetIndexAction getActionInstance() {
+        return GetIndexAction.INSTANCE;
+    }
+
+    @Override
+    protected Request.Builder createHttpRequest(String url, GetIndexRequest getIndexRequest) throws IOException {
+        List<String> list =  getIndexRequest.indices().length == 0 ?
+                List.of("*") : Arrays.asList(getIndexRequest.indices());
+        String command = "/" + String.join(",", list);
+        logger.info("command = " + command);
+        return newGetRequest(url, command);
+    }
+
+    @Override
+    protected CheckedFunction<XContentParser, GetIndexResponse, IOException> entityParser(HttpResponse httpResponse) {
+        return GetIndexResponse::fromXContent;
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/action/search/HttpClearScrollAction.java

@@ -0,0 +1,35 @@
+package org.xbib.elx.http.action.search;
+
+import org.elasticsearch.action.search.ClearScrollAction;
+import org.elasticsearch.action.search.ClearScrollRequest;
+import org.elasticsearch.action.search.ClearScrollResponse;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.xbib.elx.http.HttpAction;
+import org.xbib.netty.http.client.api.Request;
+import org.xbib.netty.http.common.HttpResponse;
+import java.io.IOException;
+
+public class HttpClearScrollAction extends HttpAction<ClearScrollRequest, ClearScrollResponse> {
+
+    @Override
+    public ClearScrollAction getActionInstance() {
+        return ClearScrollAction.INSTANCE;
+    }
+
+    @Override
+    protected Request.Builder createHttpRequest(String baseUrl, ClearScrollRequest request) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        request.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        return newDeleteRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder));
+    }
+
+    @Override
+    protected CheckedFunction<XContentParser, ClearScrollResponse, IOException> entityParser(HttpResponse httpResponse) {
+        return ClearScrollResponse::fromXContent;
+    }
+}

elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchAction.java

@@ -5,6 +5,7 @@ import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.common.CheckedFunction;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.Scroll;
 import org.xbib.elx.http.HttpAction;
 import org.xbib.netty.http.client.api.Request;
 import org.xbib.netty.http.common.HttpResponse;
@@ -20,9 +21,10 @@ public class HttpSearchAction extends HttpAction<SearchRequest, SearchResponse>
 
     @Override
     protected Request.Builder createHttpRequest(String url, SearchRequest request) {
-        // request.indices() always empty array
+        Scroll scroll = request.scroll();
+        String params = scroll != null ? "?scroll=" + scroll.keepAlive() : "";
         String index = request.indices() != null ? String.join(",", request.indices()) + "/" : "";
-        return newPostRequest(url, index + "_search", request.source().toString());
+        return newPostRequest(url, index + "_search" + params, request.source().toString());
     }
 
     @Override

elx-http/src/main/java/org/xbib/elx/http/action/search/HttpSearchScrollAction.java

@@ -0,0 +1,35 @@
+package org.xbib.elx.http.action.search;
+
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollAction;
+import org.elasticsearch.action.search.SearchScrollRequest;
+import org.elasticsearch.common.CheckedFunction;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.xbib.elx.http.HttpAction;
+import org.xbib.netty.http.client.api.Request;
+import org.xbib.netty.http.common.HttpResponse;
+import java.io.IOException;
+
+public class HttpSearchScrollAction extends HttpAction<SearchScrollRequest, SearchResponse> {
+
+    @Override
+    public SearchScrollAction getActionInstance() {
+        return SearchScrollAction.INSTANCE;
+    }
+
+    @Override
+    protected Request.Builder createHttpRequest(String baseUrl, SearchScrollRequest request) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder();
+        request.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        return newPostRequest(baseUrl, "_search/scroll", BytesReference.bytes(builder));
+    }
+
+    @Override
+    protected CheckedFunction<XContentParser, SearchResponse, IOException> entityParser(HttpResponse httpResponse) {
+        return SearchResponse::fromXContent;
+    }
+}

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.http.HttpAdminClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.http.HttpBulkClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.http.ExtendedHttpClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.http.HttpSearchClientProvider

elx-http/src/main/resources/META-INF/services/org.xbib.elx.http.HttpAction

@@ -8,12 +8,15 @@ org.xbib.elx.http.action.admin.indices.alias.get.HttpGetAliasAction
 org.xbib.elx.http.action.admin.indices.create.HttpCreateIndexAction
 org.xbib.elx.http.action.admin.indices.delete.HttpDeleteIndexAction
 org.xbib.elx.http.action.admin.indices.exists.indices.HttpIndicesExistsAction
+org.xbib.elx.http.action.admin.indices.get.HtppGetIndexAction
 org.xbib.elx.http.action.admin.indices.refresh.HttpRefreshIndexAction
 org.xbib.elx.http.action.admin.indices.settings.get.HttpGetSettingsAction
 org.xbib.elx.http.action.admin.indices.settings.put.HttpUpdateSettingsAction
 org.xbib.elx.http.action.bulk.HttpBulkAction
 org.xbib.elx.http.action.index.HttpIndexAction
+org.xbib.elx.http.action.search.HttpClearScrollAction
 org.xbib.elx.http.action.search.HttpSearchAction
+org.xbib.elx.http.action.search.HttpSearchScrollAction
 org.xbib.elx.http.action.main.HttpMainAction
 org.xbib.elx.http.action.get.HttpExistsAction
 org.xbib.elx.http.action.get.HttpGetAction

elx-http/src/test/java/org/xbib/elx/http/test/BulkClientTest.java

@@ -0,0 +1,182 @@
+package org.xbib.elx.http.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.http.HttpAdminClient;
+import org.xbib.elx.http.HttpAdminClientProvider;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ExtendWith(TestExtension.class)
+class BulkClientTest {
+
+    private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getSimpleName());
+
+    private static final Long ACTIONS = 100L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
+
+    private final TestExtension.Helper helper;
+
+    BulkClientTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testSingleDoc() throws Exception {
+        final HttpBulkClient client = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
+                .build();
+        try {
+            client.newIndex("test");
+            client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            client.flush();
+            client.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(1, client.getBulkMetric().getSucceeded().getCount());
+            if (client.getBulkController().getLastBulkError() != null) {
+                logger.error("error", client.getBulkController().getLastBulkError());
+            }
+            assertNull(client.getBulkController().getLastBulkError());
+            client.close();
+        }
+    }
+
+    @Test
+    void testNewIndex() throws Exception {
+        final HttpBulkClient client = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
+                .build();
+        client.newIndex("test");
+        client.close();
+    }
+
+    @Test
+    void testMapping() throws Exception {
+        try (HttpAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(HttpAdminClientProvider.class)
+                .put(helper.getHttpSettings())
+                .build();
+             HttpBulkClient bulkClient = ClientBuilder.builder()
+                     .setBulkClientProvider(HttpBulkClientProvider.class)
+                     .put(helper.getHttpSettings())
+                     .build()) {
+            XContentBuilder builder = JsonXContent.contentBuilder()
+                    .startObject()
+                    .startObject("properties")
+                    .startObject("location")
+                    .field("type", "geo_point")
+                    .endObject()
+                    .endObject()
+                    .endObject();
+            bulkClient.newIndex("test", Settings.EMPTY, builder);
+            assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
+        }
+    }
+
+    @Test
+    void testRandomDocs() throws Exception {
+        long numactions = ACTIONS;
+        final HttpBulkClient client = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            client.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            client.flush();
+            client.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
+            if (client.getBulkController().getLastBulkError() != null) {
+                logger.error("error", client.getBulkController().getLastBulkError());
+            }
+            assertNull(client.getBulkController().getLastBulkError());
+            client.refreshIndex("test");
+            assertEquals(numactions, client.getSearchableDocs("test"));
+            client.close();
+        }
+    }
+
+    @Test
+    void testThreadedRandomDocs() throws Exception {
+        int maxthreads = Runtime.getRuntime().availableProcessors();
+        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
+        final long actions = ACTIONS;
+        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
+        final HttpBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            Settings settings = Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .build();
+            bulkClient.newIndex("test", settings);
+            bulkClient.startBulk("test", 0, 1000);
+            logger.info("index created");
+            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+            final CountDownLatch latch = new CountDownLatch(maxthreads);
+            for (int i = 0; i < maxthreads; i++) {
+                executorService.execute(() -> {
+                    for (int i1 = 0; i1 < actions; i1++) {
+                        bulkClient.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                    }
+                    latch.countDown();
+                });
+            }
+            logger.info("waiting for latch...");
+            if (latch.await(60L, TimeUnit.SECONDS)) {
+                logger.info("flush...");
+                bulkClient.flush();
+                bulkClient.waitForResponses(60L, TimeUnit.SECONDS);
+                logger.info("got all responses, executor service shutdown...");
+                executorService.shutdown();
+                executorService.awaitTermination(60L, TimeUnit.SECONDS);
+                logger.info("pool is shut down");
+            } else {
+                logger.warn("latch timeout");
+            }
+            bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
+            assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        } finally {
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+}

elx-http/src/test/java/org/xbib/elx/http/test/ClientTest.java

@@ -1,244 +0,0 @@
-package org.xbib.elx.http.test;
-
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.common.Parameters;
-import org.xbib.elx.http.ExtendedHttpClient;
-import org.xbib.elx.http.ExtendedHttpClientProvider;
-
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-@ExtendWith(TestExtension.class)
-class ClientTest {
-
-    private static final Logger logger = LogManager.getLogger(ClientTest.class.getSimpleName());
-
-    private static final Long ACTIONS = 100L;
-
-    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
-
-    private final TestExtension.Helper helper;
-
-    ClientTest(TestExtension.Helper helper) {
-        this.helper = helper;
-    }
-
-    @Test
-    void testSingleDoc() throws Exception {
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put(helper.getHttpSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
-                .build();
-        try {
-            client.newIndex("test");
-            client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.close();
-        }
-    }
-
-    @Test
-    void testNewIndex() throws Exception {
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put(helper.getHttpSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        client.newIndex("test");
-        client.close();
-    }
-
-    @Test
-    void testNewIndexWithSettings() throws Exception {
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put(helper.getHttpSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
-        client.newIndex("test", settings);
-        GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices("test");
-        GetSettingsResponse getSettingsResponse =
-                client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
-        logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        client.close();
-    }
-
-    @Test
-    void testNewIndexWithSettingsAndMappings() throws Exception {
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put(helper.getHttpSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
-        XContentBuilder builder = JsonXContent.contentBuilder()
-                .startObject()
-                .field("date_detection", false)
-                .startObject("properties")
-                .startObject("pos")
-                .field("type", "geo_point")
-                .endObject()
-                .endObject()
-                .endObject();
-        client.newIndex("test", settings, builder);
-        GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices("test");
-        GetSettingsResponse getSettingsResponse =
-                client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
-        assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
-                .indices("test");
-        GetMappingsResponse getMappingsResponse =
-                client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
-        assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
-        client.close();
-    }
-
-    @Test
-    void testRandomDocs() throws Exception {
-        long numactions = ACTIONS;
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put(helper.getHttpSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            client.newIndex("test");
-            for (int i = 0; i < ACTIONS; i++) {
-                client.index("test", null, false, "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-            }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        } finally {
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test");
-            SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
-                    .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
-            assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value);
-            client.close();
-        }
-    }
-
-    @Test
-    void testThreadedRandomDocs() throws Exception {
-        int maxthreads = Runtime.getRuntime().availableProcessors();
-        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
-        final long actions = ACTIONS;
-        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
-                .put(helper.getHttpSettings())
-                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            Settings settings = Settings.builder()
-                    .put("index.number_of_shards", 1)
-                    .put("index.number_of_replicas", 0)
-                    .build();
-            client.newIndex("test", settings, (String)null)
-                    .startBulk("test", 0, 1000);
-            logger.info("index created");
-            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
-            final CountDownLatch latch = new CountDownLatch(maxthreads);
-            for (int i = 0; i < maxthreads; i++) {
-                executorService.execute(() -> {
-                    for (int i1 = 0; i1 < actions; i1++) {
-                        client.index("test", null, false,"{ \"name\" : \"" + helper.randomString(32) + "\"}");
-                    }
-                    latch.countDown();
-                });
-            }
-            logger.info("waiting for latch...");
-            if (latch.await(60L, TimeUnit.SECONDS)) {
-                logger.info("flush...");
-                client.flush();
-                client.waitForResponses(60L, TimeUnit.SECONDS);
-                logger.info("got all responses, executor service shutdown...");
-                executorService.shutdown();
-                executorService.awaitTermination(60L, TimeUnit.SECONDS);
-                logger.info("pool is shut down");
-            } else {
-                logger.warn("latch timeout");
-            }
-            client.stopBulk("test", 30L, TimeUnit.SECONDS);
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        } finally {
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            logger.log(Level.INFO, "refreshing index test");
-            client.refreshIndex("test");
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0)
-                    .trackTotalHits(true);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices("test")
-                    .source(builder);
-            SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
-            client.close();
-            client.close();
-        }
-    }
-}

elx-http/src/test/java/org/xbib/elx/http/test/DuplicateIDTest.java

@@ -2,18 +2,12 @@ package org.xbib.elx.http.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.common.Parameters;
-import org.xbib.elx.http.ExtendedHttpClient;
-import org.xbib.elx.http.ExtendedHttpClientProvider;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -26,10 +20,10 @@ class DuplicateIDTest {
 
     private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getSimpleName());
 
-    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
-
     private static final Long ACTIONS = 100L;
 
+    private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
+
     private final TestExtension.Helper helper;
 
     DuplicateIDTest(TestExtension.Helper helper) {
@@ -39,8 +33,8 @@ class DuplicateIDTest {
     @Test
     void testDuplicateDocIDs() throws Exception {
         long numactions = ACTIONS;
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
+        final HttpBulkClient client = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
                 .put(helper.getHttpSettings())
                 .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
                 .build();
@@ -53,23 +47,11 @@ class DuplicateIDTest {
             client.flush();
             client.waitForResponses(30L, TimeUnit.SECONDS);
             client.refreshIndex("test");
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0)
-                    .trackTotalHits(true);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices("test")
-                    .source(builder);
-            SearchResponse searchResponse =
-                    helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            long hits = searchResponse.getHits().getTotalHits().value;
-            logger.info("hits = {}", hits);
+            long hits = client.getSearchableDocs("test");
             assertTrue(hits < ACTIONS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
             client.close();
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
+            assertEquals(numactions, client.getBulkMetric().getSucceeded().getCount());
             if (client.getBulkController().getLastBulkError() != null) {
                 logger.error("error", client.getBulkController().getLastBulkError());
             }

elx-http/src/test/java/org/xbib/elx/http/test/IndexPruneTest.java

@@ -2,17 +2,15 @@ package org.xbib.elx.http.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexPruneResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.http.ExtendedHttpClient;
-import org.xbib.elx.http.ExtendedHttpClientProvider;
+import org.xbib.elx.http.HttpAdminClient;
+import org.xbib.elx.http.HttpAdminClientProvider;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,50 +35,49 @@ class IndexPruneTest {
 
     @Test
     void testPrune() throws IOException {
-        final ExtendedHttpClient client = ClientBuilder.builder(helper.client("1"))
+        final HttpAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(HttpAdminClientProvider.class)
+                .put(helper.getHttpSettings())
+                .build();
+        final HttpBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
                 .put(helper.getHttpSettings())
-                .provider(ExtendedHttpClientProvider.class)
                 .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test1", settings);
-            client.shiftIndex("test", "test1", Collections.emptyList());
-            client.newIndex("test2", settings);
-            client.shiftIndex("test", "test2", Collections.emptyList());
-            client.newIndex("test3", settings);
-            client.shiftIndex("test", "test3", Collections.emptyList());
-            client.newIndex("test4", settings);
-            client.shiftIndex("test", "test4", Collections.emptyList());
+            bulkClient.newIndex("test1", settings);
+            adminClient.shiftIndex("test", "test1", Collections.emptyList());
+            bulkClient.newIndex("test2", settings);
+            adminClient.shiftIndex("test", "test2", Collections.emptyList());
+            bulkClient.newIndex("test3", settings);
+            adminClient.shiftIndex("test", "test3", Collections.emptyList());
+            bulkClient.newIndex("test4", settings);
+            adminClient.shiftIndex("test", "test4", Collections.emptyList());
             IndexPruneResult indexPruneResult =
-                    client.pruneIndex("test", "test4", 2, 2, true);
+                    adminClient.pruneIndex("test", "test4", 2, 2, true);
             assertTrue(indexPruneResult.getDeletedIndices().contains("test1"));
             assertTrue(indexPruneResult.getDeletedIndices().contains("test2"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test3"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test4"));
             List<Boolean> list = new ArrayList<>();
             for (String index : Arrays.asList("test1", "test2", "test3", "test4")) {
-                IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
-                indicesExistsRequest.indices(index);
-                IndicesExistsResponse indicesExistsResponse =
-                        client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
-                logger.info("indices exists response for {} is {}", index, indicesExistsResponse.isExists());
-                list.add(indicesExistsResponse.isExists());
+                list.add(bulkClient.isIndexExists(index));
             }
+            logger.info(list);
             assertFalse(list.get(0));
             assertFalse(list.get(1));
             assertTrue(list.get(2));
             assertTrue(list.get(3));
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            adminClient.close();
         }
     }
 }

elx-http/src/test/java/org/xbib/elx/http/test/IndexShiftTest.java

@@ -4,16 +4,16 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexShiftResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.http.ExtendedHttpClient;
-import org.xbib.elx.http.ExtendedHttpClientProvider;
+import org.xbib.elx.http.HttpAdminClient;
+import org.xbib.elx.http.HttpAdminClientProvider;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -35,56 +35,54 @@ class IndexShiftTest {
 
     @Test
     void testIndexShift() throws Exception {
-        final ExtendedHttpClient client = ClientBuilder.builder()
+        final HttpAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(HttpAdminClientProvider.class)
+                .put(helper.getHttpSettings())
+                .build();
+        final HttpBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
                 .put(helper.getHttpSettings())
-                .provider(ExtendedHttpClientProvider.class)
                 .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test1234", settings);
+            bulkClient.newIndex("test1234", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test1234", helper.randomString(1), false,
+                bulkClient.index("test1234", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
             IndexShiftResult indexShiftResult =
-                    client.shiftIndex("test", "test1234",  Arrays.asList("a", "b", "c"));
-
+                    adminClient.shiftIndex("test", "test1234",  Arrays.asList("a", "b", "c"));
             assertTrue(indexShiftResult.getNewAliases().contains("a"));
             assertTrue(indexShiftResult.getNewAliases().contains("b"));
             assertTrue(indexShiftResult.getNewAliases().contains("c"));
             assertTrue(indexShiftResult.getMovedAliases().isEmpty());
-
-            Map<String, String> aliases = client.getAliases("test1234");
+            Map<String, String> aliases = adminClient.getAliases("test1234");
             logger.log(Level.DEBUG, "aliases = " + aliases);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("test"));
-
-            String resolved = client.resolveAlias("test");
+            String resolved = adminClient.resolveAlias("test");
             logger.log(Level.DEBUG, "resolved = " + resolved);
-            aliases = client.getAliases(resolved);
+            aliases = adminClient.getAliases(resolved);
             logger.log(Level.DEBUG, "aliases = " + aliases);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("test"));
-
-            client.newIndex("test5678", settings);
+            bulkClient.newIndex("test5678", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test5678", helper.randomString(1), false,
+                bulkClient.index("test5678", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-
-            indexShiftResult = client.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+            indexShiftResult = adminClient.shiftIndex("test", "test5678", Arrays.asList("d", "e", "f"),
                     (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
                             .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
             );
@@ -94,31 +92,28 @@ class IndexShiftTest {
             assertTrue(indexShiftResult.getMovedAliases().contains("a"));
             assertTrue(indexShiftResult.getMovedAliases().contains("b"));
             assertTrue(indexShiftResult.getMovedAliases().contains("c"));
-
-            aliases = client.getAliases("test5678");
+            aliases = adminClient.getAliases("test5678");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-
-            resolved = client.resolveAlias("test");
-            aliases = client.getAliases(resolved);
+            resolved = adminClient.resolveAlias("test");
+            aliases = adminClient.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            adminClient.close();
         }
     }
 }

elx-http/src/test/java/org/xbib/elx/http/test/SearchTest.java

@@ -0,0 +1,82 @@
+package org.xbib.elx.http.test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
+import org.xbib.elx.http.HttpSearchClient;
+import org.xbib.elx.http.HttpSearchClientProvider;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+@ExtendWith(TestExtension.class)
+class SearchTest {
+
+    private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
+
+    private static final Long ACTIONS = 100L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
+
+    private final TestExtension.Helper helper;
+
+    SearchTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testDocStream() throws Exception {
+        long numactions = ACTIONS;
+        final HttpBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(HttpBulkClientProvider.class)
+                .put(helper.getHttpSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .build();
+        try (bulkClient) {
+            bulkClient.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                bulkClient.index("test", null, false,
+                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+            bulkClient.refreshIndex("test");
+            assertEquals(numactions, bulkClient.getSearchableDocs("test"));
+        }
+        assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        if (bulkClient.getBulkController().getLastBulkError() != null) {
+            logger.error("error", bulkClient.getBulkController().getLastBulkError());
+        }
+        assertNull(bulkClient.getBulkController().getLastBulkError());
+        try (HttpSearchClient searchClient = ClientBuilder.builder()
+                .setSearchClientProvider(HttpSearchClientProvider.class)
+                .put(helper.getHttpSettings())
+                .build()) {
+            Stream<SearchHit> stream = searchClient.search(qb -> qb
+                            .setIndices("test")
+                            .setQuery(QueryBuilders.matchAllQuery()),
+                    TimeValue.timeValueMinutes(1), 10);
+            long count = stream.count();
+            assertEquals(numactions, count);
+            Stream<String> ids = searchClient.getIds(qb -> qb
+                    .setIndices("test")
+                    .setQuery(QueryBuilders.matchAllQuery()));
+            final AtomicInteger idcount = new AtomicInteger();
+            ids.forEach(id -> {
+                logger.info(id);
+                idcount.incrementAndGet();
+            });
+            assertEquals(numactions, idcount.get());
+        }
+    }
+}

elx-http/src/test/java/org/xbib/elx/http/test/SmokeTest.java

@@ -2,14 +2,15 @@ package org.xbib.elx.http.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexDefinition;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.http.ExtendedHttpClient;
-import org.xbib.elx.http.ExtendedHttpClientProvider;
+import org.xbib.elx.http.HttpAdminClient;
+import org.xbib.elx.http.HttpAdminClientProvider;
+import org.xbib.elx.http.HttpBulkClient;
+import org.xbib.elx.http.HttpBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -29,47 +30,46 @@ class SmokeTest {
 
     @Test
     void smokeTest() throws Exception {
-        final ExtendedHttpClient client = ClientBuilder.builder()
-                .provider(ExtendedHttpClientProvider.class)
+        try (HttpAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(HttpAdminClientProvider.class)
                 .put(helper.getHttpSettings())
                 .build();
-        try {
-            assertEquals(helper.getClusterName(), client.getClusterName());
-            client.newIndex("test_smoke");
-            client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
-            client.flush();
-            client.waitForResponses(30, TimeUnit.SECONDS);
-            client.checkMapping("test_smoke");
-            client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
-            client.delete("test_smoke", "1");
-            client.flush();
-            client.waitForResponses(30, TimeUnit.SECONDS);
-            client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
-            client.delete("test_smoke", "1");
-            client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
-            client.flush();
-            client.deleteIndex("test_smoke");
-            IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
-                    .build());
+             HttpBulkClient bulkClient = ClientBuilder.builder()
+                     .setBulkClientProvider(HttpBulkClientProvider.class)
+                     .put(helper.getHttpSettings())
+                     .build()) {
+            IndexDefinition indexDefinition =
+                    adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
             assertEquals(0, indexDefinition.getReplicaLevel());
-            client.newIndex(indexDefinition);
-            client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
-            client.flush();
-            client.waitForResponses(30, TimeUnit.SECONDS);
-            client.updateReplicaLevel(indexDefinition, 2);
-            int replica = client.getReplicaLevel(indexDefinition);
+            assertEquals(helper.getClusterName(), adminClient.getClusterName());
+            bulkClient.newIndex("test_smoke");
+            bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            adminClient.checkMapping("test_smoke");
+            bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
+            bulkClient.delete("test_smoke", "1");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
+            bulkClient.delete("test_smoke", "1");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            adminClient.deleteIndex("test_smoke");
+            bulkClient.newIndex(indexDefinition);
+            bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            adminClient.updateReplicaLevel(indexDefinition, 2);
+            int replica = adminClient.getReplicaLevel(indexDefinition);
             assertEquals(2, replica);
-            client.deleteIndex(indexDefinition);
-            assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
-            assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
+            assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            adminClient.deleteIndex(indexDefinition);
         }
     }
 }

elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClient.java

@@ -1,69 +0,0 @@
-package org.xbib.elx.node;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.env.Environment;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.plugins.Plugin;
-import org.xbib.elx.common.AbstractExtendedClient;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-public class ExtendedNodeClient extends AbstractExtendedClient {
-
-    private static final Logger logger = LogManager.getLogger(ExtendedNodeClient.class.getName());
-
-    private Node node;
-
-    @Override
-    protected ElasticsearchClient createClient(Settings settings) throws IOException {
-        if (settings == null) {
-            return null;
-        }
-        String version = System.getProperty("os.name")
-                + " " + System.getProperty("java.vm.name")
-                + " " + System.getProperty("java.vm.vendor")
-                + " " + System.getProperty("java.runtime.version")
-                + " " + System.getProperty("java.vm.version");
-        Settings effectiveSettings = Settings.builder().put(settings)
-                .put("node.client", true)
-                .put("node.master", false)
-                .put("node.data", false)
-                .build();
-        XContentBuilder builder = XContentFactory.jsonBuilder();
-        effectiveSettings.toXContent(builder, new ToXContent.MapParams(Collections.singletonMap("flat_settings", "true")));
-        logger.info("creating node client on {} with effective settings {}",
-                version, Strings.toString(builder));
-        Collection<Class<? extends Plugin>> plugins = Collections.emptyList();
-        this.node = new BulkNode(new Environment(effectiveSettings, null), plugins);
-        try {
-            node.start();
-        } catch (Exception e) {
-            throw new IOException(e);
-        }
-        return node.client();
-    }
-
-    @Override
-    protected void closeClient() throws IOException {
-        if (node != null) {
-            logger.debug("closing node client");
-            node.close();
-        }
-    }
-
-    private static class BulkNode extends Node {
-
-        BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
-            super(env, classpathPlugins, false);
-        }
-    }
-}

elx-node/src/main/java/org/xbib/elx/node/ExtendedNodeClientProvider.java

@@ -1,10 +0,0 @@
-package org.xbib.elx.node;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class ExtendedNodeClientProvider implements ExtendedClientProvider<ExtendedNodeClient> {
-    @Override
-    public ExtendedNodeClient getExtendedClient() {
-        return new ExtendedNodeClient();
-    }
-}

elx-node/src/main/java/org/xbib/elx/node/NodeAdminClient.java

@@ -0,0 +1,25 @@
+package org.xbib.elx.node;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractAdminClient;
+import java.io.IOException;
+
+public class NodeAdminClient extends AbstractAdminClient {
+
+    private final NodeClientHelper helper;
+
+    public NodeAdminClient() {
+        this.helper = new NodeClientHelper();
+    }
+
+    @Override
+    public ElasticsearchClient createClient(Settings settings) {
+        return helper.createClient(settings, null);
+    }
+
+    @Override
+    public void closeClient(Settings settings) throws IOException {
+        helper.closeClient(settings);
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeAdminClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.node;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class NodeAdminClientProvider implements AdminClientProvider<NodeAdminClient> {
+
+    @Override
+    public NodeAdminClient getClient() {
+        return new NodeAdminClient();
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeBulkClient.java

@@ -0,0 +1,25 @@
+package org.xbib.elx.node;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractBulkClient;
+import java.io.IOException;
+
+public class NodeBulkClient extends AbstractBulkClient {
+
+    private final NodeClientHelper helper;
+
+    public NodeBulkClient() {
+        this.helper = new NodeClientHelper();
+    }
+
+    @Override
+    public ElasticsearchClient createClient(Settings settings) {
+        return helper.createClient(settings, null);
+    }
+
+    @Override
+    public void closeClient(Settings settings) throws IOException {
+        helper.closeClient(settings);
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.node;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class NodeBulkClientProvider implements BulkClientProvider<NodeBulkClient> {
+
+    @Override
+    public NodeBulkClient getClient() {
+        return new NodeBulkClient();
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeClientHelper.java

@@ -0,0 +1,98 @@
+package org.xbib.elx.node;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.common.network.NetworkModule;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.transport.Netty4Plugin;
+import org.xbib.elx.common.Parameters;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class NodeClientHelper {
+
+    private static final Logger logger = LogManager.getLogger(NodeClientHelper.class.getName());
+
+    private static Object configurationObject;
+
+    private static Node node;
+
+    private static final Map<String, ElasticsearchClient> clientMap = new HashMap<>();
+
+    public ElasticsearchClient createClient(Settings settings, Object object) {
+        if (configurationObject == null) {
+            configurationObject = object;
+        }
+        if (configurationObject instanceof ElasticsearchClient) {
+            return (ElasticsearchClient) configurationObject;
+        }
+        return clientMap.computeIfAbsent(settings.get("cluster.name"),
+                key -> innerCreateClient(settings));
+    }
+
+    public void closeClient(Settings settings) throws IOException {
+        ElasticsearchClient client = clientMap.remove(settings.get("cluster.name"));
+        if (client != null) {
+            logger.debug("closing node...");
+            node.close();
+            node = null;
+        }
+    }
+
+    private ElasticsearchClient innerCreateClient(Settings settings) {
+        String version = System.getProperty("os.name")
+                + " " + System.getProperty("java.vm.name")
+                + " " + System.getProperty("java.vm.vendor")
+                + " " + System.getProperty("java.runtime.version")
+                + " " + System.getProperty("java.vm.version");
+        Settings effectiveSettings = Settings.builder()
+                .put(settings.filter(key -> !isPrivateSettings(key)))
+                .put("node.master", false)
+                .put("node.data", false)
+                // "node.processors"
+                .put(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
+                        settings.get(EsExecutors.NODE_PROCESSORS_SETTING.getKey(),
+                                String.valueOf(Runtime.getRuntime().availableProcessors())))
+                // "transport.type"
+                .put(NetworkModule.TRANSPORT_TYPE_KEY,
+                        Netty4Plugin.NETTY_TRANSPORT_NAME)
+                .build();
+        logger.info("creating node client on {} with effective settings {}",
+                version, effectiveSettings.toDelimitedString(','));
+        Collection<Class<? extends Plugin>> plugins =
+                Collections.singletonList(Netty4Plugin.class);
+        node = new BulkNode(new Environment(effectiveSettings, null), plugins);
+        try {
+            node.start();
+            return node.client();
+        } catch (NodeValidationException e) {
+            logger.log(Level.ERROR, e.getMessage(), e);
+        }
+        return null;
+    }
+
+    private static boolean isPrivateSettings(String key) {
+        return key.equals(Parameters.MAX_ACTIONS_PER_REQUEST.name()) ||
+                key.equals(Parameters.MAX_CONCURRENT_REQUESTS.name()) ||
+                key.equals(Parameters.MAX_VOLUME_PER_REQUEST.name()) ||
+                key.equals(Parameters.FLUSH_INTERVAL.name());
+    }
+
+    private static class BulkNode extends Node {
+
+        BulkNode(Environment env, Collection<Class<? extends Plugin>> classpathPlugins) {
+            super(env, classpathPlugins, false);
+        }
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeSearchClient.java

@@ -0,0 +1,25 @@
+package org.xbib.elx.node;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractSearchClient;
+import java.io.IOException;
+
+public class NodeSearchClient extends AbstractSearchClient {
+
+    private final NodeClientHelper helper;
+
+    public NodeSearchClient() {
+        this.helper = new NodeClientHelper();
+    }
+
+    @Override
+    public ElasticsearchClient createClient(Settings settings) {
+        return helper.createClient(settings, null);
+    }
+
+    @Override
+    public void closeClient(Settings settings) throws IOException {
+        helper.closeClient(settings);
+    }
+}

elx-node/src/main/java/org/xbib/elx/node/NodeSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.node;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class NodeSearchClientProvider implements SearchClientProvider<NodeSearchClient> {
+
+    @Override
+    public NodeSearchClient getClient() {
+        return new NodeSearchClient();
+    }
+}

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.node.NodeAdminClientProvider

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.node.NodeBulkClientProvider

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.node.ExtendedNodeClientProvider

elx-node/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.node.NodeSearchClientProvider

elx-node/src/test/java/org/xbib/elx/node/test/BulkClientTest.java

@@ -0,0 +1,180 @@
+package org.xbib.elx.node.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ExtendWith(TestExtension.class)
+class BulkClientTest {
+
+    private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
+
+    private static final Long ACTIONS = 10000L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
+
+    private final TestExtension.Helper helper;
+
+    BulkClientTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testSingleDoc() throws Exception {
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testNewIndex() throws Exception {
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
+                .build();
+        bulkClient.newIndex("test");
+        bulkClient.close();
+    }
+
+    @Test
+    void testMapping() throws Exception {
+        try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
+                .put(helper.getNodeSettings())
+                .build();
+             NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                     .setBulkClientProvider(NodeBulkClientProvider.class)
+                     .put(helper.getNodeSettings())
+                     .build()) {
+            XContentBuilder builder = JsonXContent.contentBuilder()
+                    .startObject()
+                    .startObject("properties")
+                    .startObject("location")
+                    .field("type", "geo_point")
+                    .endObject()
+                    .endObject()
+                    .endObject();
+            bulkClient.newIndex("test", Settings.EMPTY, builder);
+            assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
+        }
+    }
+
+    @Test
+    void testRandomDocs() throws Exception {
+        long numactions = ACTIONS;
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                bulkClient.index("test", null, false,
+                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(numactions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testThreadedRandomDocs() throws Exception {
+        int maxthreads = Runtime.getRuntime().availableProcessors();
+        final long actions = ACTIONS;
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
+                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .build();
+        try {
+            Settings settings = Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .build();
+            bulkClient.newIndex("test", settings);
+            bulkClient.startBulk("test", 0, 1000);
+            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+            final CountDownLatch latch = new CountDownLatch(maxthreads);
+            for (int i = 0; i < maxthreads; i++) {
+                executorService.execute(() -> {
+                    for (int i1 = 0; i1 < actions; i1++) {
+                        bulkClient.index("test", null, false,
+                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                    }
+                    latch.countDown();
+                });
+            }
+            logger.info("waiting for latch...");
+            if (latch.await(30L, TimeUnit.SECONDS)) {
+                logger.info("flush...");
+                bulkClient.flush();
+                bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+                logger.info("got all responses, executor service shutdown...");
+                executorService.shutdown();
+                executorService.awaitTermination(30L, TimeUnit.SECONDS);
+                logger.info("pool is shut down");
+            } else {
+                logger.warn("latch timeout");
+            }
+            bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
+            assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        } finally {
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+}

elx-node/src/test/java/org/xbib/elx/node/test/ClientTest.java

@@ -1,236 +0,0 @@
-package org.xbib.elx.node.test;
-
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.common.Parameters;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-@ExtendWith(TestExtension.class)
-class ClientTest {
-
-    private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
-
-    private static final Long ACTIONS = 10000L;
-
-    private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
-
-    private final TestExtension.Helper helper;
-
-    ClientTest(TestExtension.Helper helper) {
-        this.helper = helper;
-    }
-
-    @Test
-    void testSingleDoc() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
-                .build();
-        try {
-            client.newIndex("test");
-            client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.close();
-        }
-    }
-
-    @Test
-    void testNewIndex() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        client.newIndex("test");
-        client.close();
-    }
-
-    @Test
-    void testNewIndexWithSettings() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
-        client.newIndex("test", settings);
-        GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices("test");
-        GetSettingsResponse getSettingsResponse =
-                client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
-        logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        client.close();
-    }
-
-    @Test
-    void testNewIndexWithSettingsAndMappings() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
-        XContentBuilder builder = JsonXContent.contentBuilder()
-                .startObject()
-                .field("date_detection", false)
-                .startObject("properties")
-                .startObject("location")
-                .field("type", "geo_point")
-                .endObject()
-                .endObject()
-                .endObject();
-        client.newIndex("test", settings, builder);
-        GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices("test");
-        GetSettingsResponse getSettingsResponse =
-                client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
-        logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
-                .indices("test");
-        GetMappingsResponse getMappingsResponse =
-                client.getClient().execute(GetMappingsAction.INSTANCE, getMappingsRequest).actionGet();
-        logger.info("mappings={}", getMappingsResponse.getMappings());
-        assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
-        client.close();
-    }
-
-    @Test
-    void testRandomDocs() throws Exception {
-        long numactions = ACTIONS;
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            client.newIndex("test");
-            for (int i = 0; i < ACTIONS; i++) {
-                client.index("test", null, false,
-                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-            }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test");
-            SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client.getClient(), SearchAction.INSTANCE)
-                    .setQuery(QueryBuilders.matchAllQuery()).setSize(0);
-            assertEquals(numactions, searchRequestBuilder.execute().actionGet().getHits().getTotalHits().value);
-            client.close();
-        }
-    }
-
-    @Test
-    void testThreadedRandomDocs() throws Exception {
-        int maxthreads = Runtime.getRuntime().availableProcessors();
-        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
-        final long actions = ACTIONS;
-        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
-                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads)
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            Settings settings = Settings.builder()
-                    .put("index.number_of_shards", 1)
-                    .put("index.number_of_replicas", 0)
-                    .build();
-            client.newIndex("test", settings, (String)null)
-                    .startBulk("test", 0, 1000);
-            logger.info("index created");
-            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
-            final CountDownLatch latch = new CountDownLatch(maxthreads);
-            for (int i = 0; i < maxthreads; i++) {
-                executorService.execute(() -> {
-                    for (int i1 = 0; i1 < actions; i1++) {
-                        client.index("test", null, false,
-                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-                    }
-                    latch.countDown();
-                });
-            }
-            logger.info("waiting for latch...");
-            if (latch.await(30L, TimeUnit.SECONDS)) {
-                logger.info("flush...");
-                client.flush();
-                client.waitForResponses(30L, TimeUnit.SECONDS);
-                logger.info("got all responses, executor service shutdown...");
-                executorService.shutdown();
-                executorService.awaitTermination(30L, TimeUnit.SECONDS);
-                logger.info("pool is shut down");
-            } else {
-                logger.warn("latch timeout");
-            }
-            client.stopBulk("test", 30L, TimeUnit.SECONDS);
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            logger.log(Level.INFO, "refreshing index test");
-            client.refreshIndex("test");
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0)
-                    .trackTotalHits(true);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices("test")
-                    .source(builder);
-            SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
-            client.close();
-        }
-    }
-}

elx-node/src/test/java/org/xbib/elx/node/test/DuplicateIDTest.java

@@ -2,18 +2,12 @@ package org.xbib.elx.node.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.common.Parameters;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -26,10 +20,10 @@ class DuplicateIDTest {
 
     private static final Logger logger = LogManager.getLogger(DuplicateIDTest.class.getName());
 
-    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
-
     private static final Long ACTIONS = 100L;
 
+    private static final Long MAX_ACTIONS_PER_REQUEST = 5L;
+
     private final TestExtension.Helper helper;
 
     DuplicateIDTest(TestExtension.Helper helper) {
@@ -39,40 +33,28 @@ class DuplicateIDTest {
     @Test
     void testDuplicateDocIDs() throws Exception {
         long numactions = ACTIONS;
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
                 .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
                 .build();
         try {
-            client.newIndex("test");
+            bulkClient.newIndex("test");
             for (int i = 0; i < ACTIONS; i++) {
-                client.index("test", helper.randomString(1), false,
+                bulkClient.index("test", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-            client.refreshIndex("test");
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0)
-                    .trackTotalHits(true);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices("test")
-                    .source(builder);
-            SearchResponse searchResponse =
-                    helper.client("1").execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            long hits = searchResponse.getHits().getTotalHits().value;
-            logger.info("hits = {}", hits);
-            assertTrue(hits < ACTIONS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+            bulkClient.refreshIndex("test");
+            assertTrue(bulkClient.getSearchableDocs("test") < ACTIONS);
         } finally {
-            client.close();
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            bulkClient.close();
+            assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/IndexPruneTest.java

@@ -2,17 +2,15 @@ package org.xbib.elx.node.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsAction;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexPruneResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -37,49 +35,49 @@ class IndexPruneTest {
 
     @Test
     void testPrune() throws IOException {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
+                .put(helper.getNodeSettings())
+                .build();
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
                 .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test_prune1", settings);
-            client.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
-            client.newIndex("test_prune2", settings);
-            client.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
-            client.newIndex("test_prune3", settings);
-            client.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
-            client.newIndex("test_prune4", settings);
-            client.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
+            bulkClient.newIndex("test_prune1", settings);
+            adminClient.shiftIndex("test_prune", "test_prune1", Collections.emptyList());
+            bulkClient.newIndex("test_prune2", settings);
+            adminClient.shiftIndex("test_prune", "test_prune2", Collections.emptyList());
+            bulkClient.newIndex("test_prune3", settings);
+            adminClient.shiftIndex("test_prune", "test_prune3", Collections.emptyList());
+            bulkClient.newIndex("test_prune4", settings);
+            adminClient.shiftIndex("test_prune", "test_prune4", Collections.emptyList());
             IndexPruneResult indexPruneResult =
-                    client.pruneIndex("test_prune", "test_prune4", 2, 2, true);
+                    adminClient.pruneIndex("test_prune", "test_prune4", 2, 2, true);
             assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune1"));
             assertTrue(indexPruneResult.getDeletedIndices().contains("test_prune2"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune3"));
             assertFalse(indexPruneResult.getDeletedIndices().contains("test_prune4"));
             List<Boolean> list = new ArrayList<>();
             for (String index : Arrays.asList("test_prune1", "test_prune2", "test_prune3", "test_prune4")) {
-                IndicesExistsRequest indicesExistsRequest = new IndicesExistsRequest();
-                indicesExistsRequest.indices(index);
-                IndicesExistsResponse indicesExistsResponse =
-                        client.getClient().execute(IndicesExistsAction.INSTANCE, indicesExistsRequest).actionGet();
-                list.add(indicesExistsResponse.isExists());
+                list.add(bulkClient.isIndexExists(index));
             }
             logger.info(list);
             assertFalse(list.get(0));
             assertFalse(list.get(1));
             assertTrue(list.get(2));
             assertTrue(list.get(3));
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            adminClient.close();
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/IndexShiftTest.java

@@ -3,15 +3,16 @@ package org.xbib.elx.node.test;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.api.IndexShiftResult;
 import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.util.Arrays;
 import java.util.Map;
@@ -33,50 +34,51 @@ class IndexShiftTest {
 
     @Test
     void testIndexShift() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        final NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
+                .put(helper.getNodeSettings())
+                .build();
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
                 .build();
         try {
             Settings settings = Settings.builder()
                     .put("index.number_of_shards", 1)
                     .put("index.number_of_replicas", 0)
                     .build();
-            client.newIndex("test_shift", settings);
+            bulkClient.newIndex("test_shift", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test_shift", helper.randomString(1), false,
+                bulkClient.index("test_shift", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
             IndexShiftResult indexShiftResult =
-                    client.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
+                    adminClient.shiftIndex("test", "test_shift", Arrays.asList("a", "b", "c"));
             assertTrue(indexShiftResult.getNewAliases().contains("a"));
             assertTrue(indexShiftResult.getNewAliases().contains("b"));
             assertTrue(indexShiftResult.getNewAliases().contains("c"));
             assertTrue(indexShiftResult.getMovedAliases().isEmpty());
-
-            Map<String, String> aliases = client.getAliases("test_shift");
+            Map<String, String> aliases = adminClient.getAliases("test_shift");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("test"));
-
-            String resolved = client.resolveAlias("test");
-            aliases = client.getAliases(resolved);
+            String resolved = adminClient.resolveAlias("test");
+            aliases = adminClient.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("test"));
-
-            client.newIndex("test_shift2", settings);
+            bulkClient.newIndex("test_shift2", settings);
             for (int i = 0; i < 1; i++) {
-                client.index("test_shift2", helper.randomString(1), false,
+                bulkClient.index("test_shift2", helper.randomString(1), false,
                         "{ \"name\" : \"" + helper.randomString(32) + "\"}");
             }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-
-            indexShiftResult = client.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+            indexShiftResult = adminClient.shiftIndex("test", "test_shift2", Arrays.asList("d", "e", "f"),
                     (request, index, alias) -> request.addAliasAction(IndicesAliasesRequest.AliasActions.add()
                             .index(index).alias(alias).filter(QueryBuilders.termQuery("my_key", alias)))
             );
@@ -86,32 +88,28 @@ class IndexShiftTest {
             assertTrue(indexShiftResult.getMovedAliases().contains("a"));
             assertTrue(indexShiftResult.getMovedAliases().contains("b"));
             assertTrue(indexShiftResult.getMovedAliases().contains("c"));
-
-            aliases = client.getAliases("test_shift2");
+            aliases = adminClient.getAliases("test_shift2");
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-
-            resolved = client.resolveAlias("test");
-            aliases = client.getAliases(resolved);
+            resolved = adminClient.resolveAlias("test");
+            aliases = adminClient.getAliases(resolved);
             assertTrue(aliases.containsKey("a"));
             assertTrue(aliases.containsKey("b"));
             assertTrue(aliases.containsKey("c"));
             assertTrue(aliases.containsKey("d"));
             assertTrue(aliases.containsKey("e"));
             assertTrue(aliases.containsKey("f"));
-
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
         } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            adminClient.close();
+            bulkClient.close();
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/SearchTest.java

@@ -0,0 +1,82 @@
+package org.xbib.elx.node.test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
+import org.xbib.elx.node.NodeSearchClient;
+import org.xbib.elx.node.NodeSearchClientProvider;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Stream;
+
+@ExtendWith(TestExtension.class)
+class SearchTest {
+
+    private static final Logger logger = LogManager.getLogger(SearchTest.class.getName());
+
+    private static final Long ACTIONS = 100L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10L;
+
+    private final TestExtension.Helper helper;
+
+    SearchTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testDocStream() throws Exception {
+        long numactions = ACTIONS;
+        final NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                .setBulkClientProvider(NodeBulkClientProvider.class)
+                .put(helper.getNodeSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .build();
+        try (bulkClient) {
+            bulkClient.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                bulkClient.index("test", null, false,
+                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+            bulkClient.refreshIndex("test");
+            assertEquals(numactions, bulkClient.getSearchableDocs("test"));
+        }
+        assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        if (bulkClient.getBulkController().getLastBulkError() != null) {
+            logger.error("error", bulkClient.getBulkController().getLastBulkError());
+        }
+        assertNull(bulkClient.getBulkController().getLastBulkError());
+        try (NodeSearchClient searchClient = ClientBuilder.builder(helper.client("1"))
+                .setSearchClientProvider(NodeSearchClientProvider.class)
+                .put(helper.getNodeSettings())
+                .build()) {
+            Stream<SearchHit> stream = searchClient.search(qb -> qb
+                            .setIndices("test")
+                            .setQuery(QueryBuilders.matchAllQuery()),
+                    TimeValue.timeValueMinutes(1), 10);
+            long count = stream.count();
+            assertEquals(numactions, count);
+            Stream<String> ids = searchClient.getIds(qb -> qb
+                    .setIndices("test")
+                    .setQuery(QueryBuilders.matchAllQuery()));
+            final AtomicInteger idcount = new AtomicInteger();
+            ids.forEach(id -> {
+                logger.info(id);
+                idcount.incrementAndGet();
+            });
+            assertEquals(numactions, idcount.get());
+        }
+    }
+}

elx-node/src/test/java/org/xbib/elx/node/test/SmokeTest.java

@@ -2,14 +2,15 @@ package org.xbib.elx.node.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.common.settings.Settings;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.xbib.elx.common.ClientBuilder;
 import org.xbib.elx.api.IndexDefinition;
-import org.xbib.elx.node.ExtendedNodeClient;
-import org.xbib.elx.node.ExtendedNodeClientProvider;
+import org.xbib.elx.node.NodeAdminClient;
+import org.xbib.elx.node.NodeAdminClientProvider;
+import org.xbib.elx.node.NodeBulkClient;
+import org.xbib.elx.node.NodeBulkClientProvider;
 
 import java.util.concurrent.TimeUnit;
 
@@ -29,46 +30,46 @@ class SmokeTest {
 
     @Test
     void smokeTest() throws Exception {
-        final ExtendedNodeClient client = ClientBuilder.builder(helper.client("1"))
-                .provider(ExtendedNodeClientProvider.class)
+        try (NodeAdminClient adminClient = ClientBuilder.builder(helper.client("1"))
+                .setAdminClientProvider(NodeAdminClientProvider.class)
+                .put(helper.getNodeSettings())
                 .build();
-        try {
-            assertEquals(helper.getClusterName(), client.getClusterName());
-            client.newIndex("test_smoke");
-            client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
-            client.flush();
-            client.waitForResponses(30, TimeUnit.SECONDS);
-            client.checkMapping("test_smoke");
-            client.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
-            client.delete("test_smoke", "1");
-            client.flush();
-            client.waitForResponses(30, TimeUnit.SECONDS);
-            client.waitForRecovery("test_smoke", 10L, TimeUnit.SECONDS);
-            client.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
-            client.delete("test_smoke", "1");
-            client.flush();
-            client.deleteIndex("test_smoke");
-            IndexDefinition indexDefinition = client.buildIndexDefinitionFromSettings("test_smoke", Settings.builder()
-                    .build());
+             NodeBulkClient bulkClient = ClientBuilder.builder(helper.client("1"))
+                     .setBulkClientProvider(NodeBulkClientProvider.class)
+                     .put(helper.getNodeSettings())
+                     .build()) {
+            IndexDefinition indexDefinition =
+                    adminClient.buildIndexDefinitionFromSettings("test_smoke", Settings.EMPTY);
             assertEquals(0, indexDefinition.getReplicaLevel());
-            client.newIndex(indexDefinition);
-            client.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
-            client.flush();
-            client.waitForResponses(30, TimeUnit.SECONDS);
-            client.updateReplicaLevel(indexDefinition, 2);
-            int replica = client.getReplicaLevel(indexDefinition);
+            assertEquals(helper.getClusterName(), adminClient.getClusterName());
+            bulkClient.newIndex("test_smoke");
+            bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            adminClient.checkMapping("test_smoke");
+            bulkClient.update("test_smoke", "1", "{ \"name\" : \"Another name\"}");
+            bulkClient.delete("test_smoke", "1");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            bulkClient.index("test_smoke", "1", true, "{ \"name\" : \"Hello World\"}");
+            bulkClient.delete("test_smoke", "1");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            adminClient.deleteIndex("test_smoke");
+            bulkClient.newIndex(indexDefinition);
+            bulkClient.index(indexDefinition.getFullIndexName(), "1", true, "{ \"name\" : \"Hello World\"}");
+            bulkClient.flush();
+            bulkClient.waitForResponses(30, TimeUnit.SECONDS);
+            adminClient.updateReplicaLevel(indexDefinition, 2);
+            int replica = adminClient.getReplicaLevel(indexDefinition);
             assertEquals(2, replica);
-            client.deleteIndex(indexDefinition);
-            assertEquals(0, client.getBulkController().getBulkMetric().getFailed().getCount());
-            assertEquals(6, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            client.close();
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
+            assertEquals(0, bulkClient.getBulkMetric().getFailed().getCount());
+            assertEquals(6, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
             }
-            assertNull(client.getBulkController().getLastBulkError());
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            adminClient.deleteIndex(indexDefinition);
         }
     }
 }

elx-node/src/test/java/org/xbib/elx/node/test/TestExtension.java

@@ -193,6 +193,7 @@ public class TestExtension implements ParameterResolver, BeforeEachCallback, Aft
                     .put("path.home", getHome())
                     .put("cluster.initial_master_nodes", "1")
                     .put("discovery.seed_hosts",  "127.0.0.1:9300")
+                    .put("node.max_local_storage_nodes", "2")
                     .build();
         }
 

elx-transport/src/main/java/org/xbib/elx/transport/ExtendedTransportClientProvider.java

@@ -1,11 +0,0 @@
-package org.xbib.elx.transport;
-
-import org.xbib.elx.api.ExtendedClientProvider;
-
-public class ExtendedTransportClientProvider implements ExtendedClientProvider<ExtendedTransportClient> {
-
-    @Override
-    public ExtendedTransportClient getExtendedClient() {
-        return new ExtendedTransportClient();
-    }
-}

elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClient.java

@@ -0,0 +1,36 @@
+package org.xbib.elx.transport;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractAdminClient;
+
+import java.io.IOException;
+
+/**
+ * Transport admin client.
+ */
+public class TransportAdminClient extends AbstractAdminClient {
+
+    private final TransportClientHelper helper;
+
+    public TransportAdminClient() {
+        this.helper = new TransportClientHelper();
+    }
+
+    @Override
+    public ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init((TransportClient) getClient(), settings);
+    }
+
+    @Override
+    public void closeClient(Settings settings) {
+        helper.closeClient(settings);
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportAdminClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.transport;
+
+import org.xbib.elx.api.AdminClientProvider;
+
+public class TransportAdminClientProvider implements AdminClientProvider<TransportAdminClient> {
+
+    @Override
+    public TransportAdminClient getClient() {
+        return new TransportAdminClient();
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClient.java

@@ -0,0 +1,35 @@
+package org.xbib.elx.transport;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractBulkClient;
+import java.io.IOException;
+
+/**
+ * Transport search client with additional methods.
+ */
+public class TransportBulkClient extends AbstractBulkClient {
+
+    private final TransportClientHelper helper;
+
+    public TransportBulkClient() {
+        this.helper = new TransportClientHelper();
+    }
+
+    @Override
+    public ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init((TransportClient) getClient(), settings);
+    }
+
+    @Override
+    public void closeClient(Settings settings) {
+        helper.closeClient(settings);
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportBulkClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.transport;
+
+import org.xbib.elx.api.BulkClientProvider;
+
+public class TransportBulkClientProvider implements BulkClientProvider<TransportBulkClient> {
+
+    @Override
+    public TransportBulkClient getClient() {
+        return new TransportBulkClient();
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportClientHelper.java

@@ -7,6 +7,7 @@ import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateRequestBuilder;
 import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ElasticsearchClient;
 import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.client.transport.TransportClient;
@@ -23,7 +24,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.transport.Netty4Plugin;
-import org.xbib.elx.common.AbstractExtendedClient;
 import org.xbib.elx.common.util.NetworkUtils;
 
 import java.io.IOException;
@@ -31,16 +31,19 @@ import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Transport client with additional methods using the BulkProcessor.
  */
-public class ExtendedTransportClient extends AbstractExtendedClient {
+public class TransportClientHelper {
 
-    private static final Logger logger = LogManager.getLogger(ExtendedTransportClient.class.getName());
+    private static final Logger logger = LogManager.getLogger(TransportClientHelper.class.getName());
+
+    private static final Map<String, ElasticsearchClient> clientMap = new HashMap<>();
 
-    @Override
     protected ElasticsearchClient createClient(Settings settings) throws IOException {
         if (settings != null) {
             String systemIdentifier = System.getProperty("os.name")
@@ -49,11 +52,9 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
                     + " " + System.getProperty("java.vm.version")
                     + " Elasticsearch " + Version.CURRENT.toString();
             Settings transportClientSettings = getTransportClientSettings(settings);
-            //XContentBuilder settingsBuilder = XContentFactory.jsonBuilder().startObject();
             XContentBuilder effectiveSettingsBuilder = XContentFactory.jsonBuilder().startObject();
             logger.log(Level.INFO, "creating transport client on {} with settings {}",
                     systemIdentifier,
-                    //Strings.toString(settings.toXContent(settingsBuilder, ToXContent.EMPTY_PARAMS).endObject()),
                     Strings.toString(transportClientSettings.toXContent(effectiveSettingsBuilder,
                             ToXContent.EMPTY_PARAMS).endObject()));
             return new MyTransportClient(transportClientSettings, Collections.singletonList(Netty4Plugin.class));
@@ -61,29 +62,22 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
         return null;
     }
 
-    @Override
-    protected void closeClient() {
-        if (getClient() != null) {
-            TransportClient client = (TransportClient) getClient();
-            client.close();
-            client.threadPool().shutdown();
+    public void closeClient(Settings settings) {
+        ElasticsearchClient client = clientMap.remove(settings.get("cluster.name"));
+        if (client != null) {
+            if (client instanceof Client) {
+                ((Client) client).close();
+            }
+            client.threadPool().shutdownNow();
         }
     }
 
-    @Override
-    public ExtendedTransportClient init(Settings settings) throws IOException {
-        super.init(settings);
-        // additional auto-connect
-        try {
+    public void init(TransportClient transportClient, Settings settings) throws IOException {
         Collection<TransportAddress> addrs = findAddresses(settings);
-            if (!connect(addrs, settings.getAsBoolean("autodiscover", false))) {
+        if (!connect(transportClient, addrs, settings.getAsBoolean("autodiscover", false))) {
             throw new NoNodeAvailableException("no cluster nodes available, check settings "
                     + settings.toString());
         }
-        } catch (IOException e) {
-            logger.error(e.getMessage(), e);
-        }
-        return this;
     }
 
     private Collection<TransportAddress> findAddresses(Settings settings) throws IOException {
@@ -112,11 +106,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
         return addresses;
     }
 
-    private boolean connect(Collection<TransportAddress> addresses, boolean autodiscover) {
-        if (getClient() == null) {
-            throw new IllegalStateException("no client present");
-        }
-        TransportClient transportClient = (TransportClient) getClient();
+    private boolean connect(TransportClient transportClient, Collection<TransportAddress> addresses, boolean autodiscover) {
         for (TransportAddress address : addresses) {
             transportClient.addTransportAddresses(address);
         }
@@ -126,7 +116,7 @@ public class ExtendedTransportClient extends AbstractExtendedClient {
             if (autodiscover) {
                 logger.debug("trying to discover all nodes...");
                 ClusterStateRequestBuilder clusterStateRequestBuilder =
-                        new ClusterStateRequestBuilder(getClient(), ClusterStateAction.INSTANCE);
+                        new ClusterStateRequestBuilder(transportClient, ClusterStateAction.INSTANCE);
                 ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
                 DiscoveryNodes discoveryNodes = clusterStateResponse.getState().getNodes();
                 for (DiscoveryNode discoveryNode : discoveryNodes) {

elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClient.java

@@ -0,0 +1,35 @@
+package org.xbib.elx.transport;
+
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.xbib.elx.common.AbstractSearchClient;
+import java.io.IOException;
+
+/**
+ * Transport search client with additional methods.
+ */
+public class TransportSearchClient extends AbstractSearchClient {
+
+    private final TransportClientHelper helper;
+
+    public TransportSearchClient() {
+        this.helper = new TransportClientHelper();
+    }
+
+    @Override
+    public ElasticsearchClient createClient(Settings settings) throws IOException {
+        return helper.createClient(settings);
+    }
+
+    @Override
+    public void init(Settings settings) throws IOException {
+        super.init(settings);
+        helper.init((TransportClient) getClient(), settings);
+    }
+
+    @Override
+    public void closeClient(Settings settings) {
+        helper.closeClient(settings);
+    }
+}

elx-transport/src/main/java/org/xbib/elx/transport/TransportSearchClientProvider.java

@@ -0,0 +1,11 @@
+package org.xbib.elx.transport;
+
+import org.xbib.elx.api.SearchClientProvider;
+
+public class TransportSearchClientProvider implements SearchClientProvider<TransportSearchClient> {
+
+    @Override
+    public TransportSearchClient getClient() {
+        return new TransportSearchClient();
+    }
+}

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.AdminClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.transport.TransportAdminClientProvider

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.BulkClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.transport.TransportBulkClientProvider

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.ExtendedClientProvider

@@ -1 +0,0 @@
-org.xbib.elx.transport.ExtendedTransportClientProvider

elx-transport/src/main/resources/META-INF/services/org.xbib.elx.api.SearchClientProvider

@@ -0,0 +1 @@
+org.xbib.elx.transport.TransportSearchClientProvider

elx-transport/src/test/java/org/xbib/elx/transport/test/BulkClientTest.java

@@ -0,0 +1,190 @@
+package org.xbib.elx.transport.test;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.xbib.elx.common.ClientBuilder;
+import org.xbib.elx.common.Parameters;
+import org.xbib.elx.transport.TransportAdminClient;
+import org.xbib.elx.transport.TransportAdminClientProvider;
+import org.xbib.elx.transport.TransportBulkClient;
+import org.xbib.elx.transport.TransportBulkClientProvider;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@ExtendWith(TestExtension.class)
+class BulkClientTest {
+
+    private static final Logger logger = LogManager.getLogger(BulkClientTest.class.getName());
+
+    private static final Long ACTIONS = 10000L;
+
+    private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
+
+    private final TestExtension.Helper helper;
+
+    BulkClientTest(TestExtension.Helper helper) {
+        this.helper = helper;
+    }
+
+    @Test
+    void testSingleDoc() throws Exception {
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            bulkClient.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } finally {
+            assertEquals(1, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testNewIndex() throws Exception {
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
+                .build();
+        bulkClient.newIndex("test");
+        bulkClient.close();
+    }
+
+    @Test
+    void testMapping() throws Exception {
+        try (TransportAdminClient adminClient = ClientBuilder.builder()
+                .setAdminClientProvider(TransportAdminClientProvider.class)
+                .put(helper.getTransportSettings())
+                .build();
+             TransportBulkClient bulkClient = ClientBuilder.builder()
+                     .setBulkClientProvider(TransportBulkClientProvider.class)
+                     .put(helper.getTransportSettings())
+                     .build()) {
+            XContentBuilder builder = JsonXContent.contentBuilder()
+                    .startObject()
+                    .startObject("properties")
+                    .startObject("location")
+                    .field("type", "geo_point")
+                    .endObject()
+                    .endObject()
+                    .endObject();
+            bulkClient.newIndex("test", Settings.EMPTY, builder);
+            assertTrue(adminClient.getMapping("test", "_doc").containsKey("properties"));
+        }
+    }
+
+    @Test
+    void testRandomDocs() throws Exception {
+        long numactions = ACTIONS;
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .build();
+        try {
+            bulkClient.newIndex("test");
+            for (int i = 0; i < ACTIONS; i++) {
+                bulkClient.index("test", null, false,
+                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+            }
+            bulkClient.flush();
+            bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            assertEquals(numactions, bulkClient.getBulkMetric().getSucceeded().getCount());
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(numactions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+
+    @Test
+    void testThreadedRandomDocs() throws Exception {
+        int maxthreads = Runtime.getRuntime().availableProcessors();
+        final long actions = ACTIONS;
+        final TransportBulkClient bulkClient = ClientBuilder.builder()
+                .setBulkClientProvider(TransportBulkClientProvider.class)
+                .put(helper.getTransportSettings())
+                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
+                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
+                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
+                .put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
+                .build();
+        try {
+            Settings settings = Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .build();
+            bulkClient.newIndex("test", settings);
+            bulkClient.startBulk("test", 0, 1000);
+            logger.info("index created");
+            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
+            final CountDownLatch latch = new CountDownLatch(maxthreads);
+            for (int i = 0; i < maxthreads; i++) {
+                executorService.execute(() -> {
+                    for (int i1 = 0; i1 < actions; i1++) {
+                        bulkClient.index("test", null, false,
+                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
+                    }
+                    latch.countDown();
+                });
+            }
+            logger.info("waiting for latch...");
+            if (latch.await(30L, TimeUnit.SECONDS)) {
+                logger.info("flush...");
+                bulkClient.flush();
+                bulkClient.waitForResponses(30L, TimeUnit.SECONDS);
+                logger.info("got all responses, executor service shutdown...");
+                executorService.shutdown();
+                executorService.awaitTermination(30L, TimeUnit.SECONDS);
+                logger.info("pool is shut down");
+            } else {
+                logger.warn("latch timeout");
+            }
+            bulkClient.stopBulk("test", 30L, TimeUnit.SECONDS);
+            assertEquals(maxthreads * actions, bulkClient.getBulkMetric().getSucceeded().getCount());
+        } catch (NoNodeAvailableException e) {
+            logger.warn("skipping, no node available");
+        } catch (Exception e) {
+            logger.error(e.getMessage(), e);
+        } finally {
+            if (bulkClient.getBulkController().getLastBulkError() != null) {
+                logger.error("error", bulkClient.getBulkController().getLastBulkError());
+            }
+            assertNull(bulkClient.getBulkController().getLastBulkError());
+            bulkClient.refreshIndex("test");
+            assertEquals(maxthreads * actions, bulkClient.getSearchableDocs("test"));
+            bulkClient.close();
+        }
+    }
+}

elx-transport/src/test/java/org/xbib/elx/transport/test/ClientTest.java

@@ -1,247 +0,0 @@
-package org.xbib.elx.transport.test;
-
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsAction;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
-import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
-import org.elasticsearch.action.search.SearchAction;
-import org.elasticsearch.action.search.SearchRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.json.JsonXContent;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.xbib.elx.common.ClientBuilder;
-import org.xbib.elx.common.Parameters;
-import org.xbib.elx.transport.ExtendedTransportClient;
-import org.xbib.elx.transport.ExtendedTransportClientProvider;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-@ExtendWith(TestExtension.class)
-class ClientTest {
-
-    private static final Logger logger = LogManager.getLogger(ClientTest.class.getName());
-
-    private static final Long ACTIONS = 10000L;
-
-    private static final Long MAX_ACTIONS_PER_REQUEST = 10000L;
-
-    private final TestExtension.Helper helper;
-
-    ClientTest(TestExtension.Helper helper) {
-        this.helper = helper;
-    }
-
-    @Test
-    void testSingleDoc() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(30))
-                .build();
-        try {
-            client.newIndex("test");
-            client.index("test", "1", true, "{ \"name\" : \"Hello World\"}"); // single doc ingest
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } finally {
-            assertEquals(1, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.close();
-        }
-    }
-
-    @Test
-    void testNewIndex() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        client.newIndex("test");
-        client.close();
-    }
-
-    @Test
-    void testNewIndexWithSettings() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
-        client.newIndex("test", settings);
-        GetSettingsRequest getSettingsRequest = new GetSettingsRequest()
-                .indices("test");
-        GetSettingsResponse getSettingsResponse =
-                client.getClient().execute(GetSettingsAction.INSTANCE, getSettingsRequest).actionGet();
-        logger.log(Level.INFO, "settings=" + getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        assertEquals("1", getSettingsResponse.getSetting("test", "index.number_of_shards"));
-        client.close();
-    }
-
-    @Test
-    void testNewIndexWithSettingsAndMapping() throws Exception {
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(5))
-                .build();
-        Settings settings = Settings.builder().put("index.number_of_shards", "1").build();
-        XContentBuilder builder = JsonXContent.contentBuilder()
-                .startObject()
-                .field("date_detection", false)
-                .startObject("properties")
-                .startObject("location")
-                .field("type", "geo_point")
-                .endObject()
-                .endObject()
-                .endObject();
-        client.newIndex("test", settings, builder);
-        GetMappingsRequest getMappingsRequest = new GetMappingsRequest()
-                .indices("test");
-        GetMappingsResponse getMappingsResponse = client.getClient()
-                .execute(GetMappingsAction.INSTANCE, getMappingsRequest)
-                .actionGet();
-        logger.info("mappings={}", getMappingsResponse.getMappings());
-        assertTrue(getMappingsResponse.getMappings().get("test").containsKey("_doc"));
-        client.close();
-    }
-
-    @Test
-    void testRandomDocs() throws Exception {
-        long numactions = ACTIONS;
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), MAX_ACTIONS_PER_REQUEST)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .build();
-        try {
-            client.newIndex("test");
-            for (int i = 0; i < ACTIONS; i++) {
-                client.index("test", null, false,
-                        "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-            }
-            client.flush();
-            client.waitForResponses(30L, TimeUnit.SECONDS);
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        } finally {
-            assertEquals(numactions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            client.refreshIndex("test");
-            SearchSourceBuilder builder = new SearchSourceBuilder();
-            builder.query(QueryBuilders.matchAllQuery());
-            builder.size(0);
-            SearchRequest searchRequest = new SearchRequest();
-            searchRequest.indices("test");
-            searchRequest.source(builder);
-            SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            logger.log(Level.INFO, searchResponse.toString());
-            assertEquals(numactions, searchResponse.getHits().getTotalHits().value);
-            client.close();
-        }
-    }
-
-    @Test
-    void testThreadedRandomDocs() throws Exception {
-        int maxthreads = Runtime.getRuntime().availableProcessors();
-        Long maxActionsPerRequest = MAX_ACTIONS_PER_REQUEST;
-        final long actions = ACTIONS;
-        logger.info("maxthreads={} maxactions={} maxloop={}", maxthreads, maxActionsPerRequest, actions);
-        final ExtendedTransportClient client = ClientBuilder.builder()
-                .provider(ExtendedTransportClientProvider.class)
-                .put(helper.getTransportSettings())
-                .put(Parameters.MAX_CONCURRENT_REQUESTS.name(), maxthreads * 2)
-                .put(Parameters.MAX_ACTIONS_PER_REQUEST.name(), maxActionsPerRequest)
-                .put(Parameters.FLUSH_INTERVAL.name(), TimeValue.timeValueSeconds(60))
-                .put(Parameters.ENABLE_BULK_LOGGING.name(), "true")
-                .build();
-        try {
-            Settings settings = Settings.builder()
-                    .put("index.number_of_shards", 1)
-                    .put("index.number_of_replicas", 0)
-                    .build();
-            client.newIndex("test", settings, (String)null)
-                    .startBulk("test", 0, 1000);
-            logger.info("index created");
-            ExecutorService executorService = Executors.newFixedThreadPool(maxthreads);
-            final CountDownLatch latch = new CountDownLatch(maxthreads);
-            for (int i = 0; i < maxthreads; i++) {
-                executorService.execute(() -> {
-                    for (int i1 = 0; i1 < actions; i1++) {
-                        client.index("test", null, false,
-                                "{ \"name\" : \"" + helper.randomString(32) + "\"}");
-                    }
-                    latch.countDown();
-                });
-            }
-            logger.info("waiting for latch...");
-            if (latch.await(30L, TimeUnit.SECONDS)) {
-                logger.info("flush...");
-                client.flush();
-                client.waitForResponses(30L, TimeUnit.SECONDS);
-                logger.info("got all responses, executor service shutdown...");
-                executorService.shutdown();
-                executorService.awaitTermination(30L, TimeUnit.SECONDS);
-                logger.info("pool is shut down");
-            } else {
-                logger.warn("latch timeout");
-            }
-            client.stopBulk("test", 30L, TimeUnit.SECONDS);
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-        } catch (NoNodeAvailableException e) {
-            logger.warn("skipping, no node available");
-        } catch (Exception e) {
-            logger.error(e.getMessage(), e);
-        } finally {
-            if (client.getBulkController().getLastBulkError() != null) {
-                logger.error("error", client.getBulkController().getLastBulkError());
-            }
-            assertNull(client.getBulkController().getLastBulkError());
-            assertEquals(maxthreads * actions, client.getBulkController().getBulkMetric().getSucceeded().getCount());
-            logger.log(Level.INFO, "refreshing index test");
-            client.refreshIndex("test");
-            SearchSourceBuilder builder = new SearchSourceBuilder()
-                    .query(QueryBuilders.matchAllQuery())
-                    .size(0)
-                    .trackTotalHits(true);
-            SearchRequest searchRequest = new SearchRequest()
-                    .indices("test")
-                    .source(builder);
-            SearchResponse searchResponse = client.getClient().execute(SearchAction.INSTANCE, searchRequest).actionGet();
-            assertEquals(maxthreads * actions, searchResponse.getHits().getTotalHits().value);
-            client.close();
-        }
-    }
-}